2024-12-08 08:13:55,827 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-08 08:13:55,838 main DEBUG Took 0.009423 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-08 08:13:55,838 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-08 08:13:55,838 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-08 08:13:55,839 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-08 08:13:55,840 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:13:55,847 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-08 08:13:55,859 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,860 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:13:55,861 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,861 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:13:55,862 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,862 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:13:55,863 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,863 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:13:55,864 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,864 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:13:55,865 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,865 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:13:55,865 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,866 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:13:55,866 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,866 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:13:55,867 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,867 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:13:55,867 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,868 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:13:55,868 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,868 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:13:55,869 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,869 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:13:55,869 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,869 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-08 08:13:55,871 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:13:55,872 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-08 08:13:55,874 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-08 08:13:55,874 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-08 08:13:55,876 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-08 08:13:55,876 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-08 08:13:55,885 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-08 08:13:55,887 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-08 08:13:55,889 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-08 08:13:55,889 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-08 08:13:55,890 main DEBUG createAppenders(={Console}) 2024-12-08 08:13:55,890 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-08 08:13:55,891 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-08 08:13:55,891 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-08 08:13:55,892 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-08 08:13:55,892 main DEBUG OutputStream closed 2024-12-08 08:13:55,892 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-08 08:13:55,893 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-08 08:13:55,893 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-08 08:13:55,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-08 08:13:55,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-08 08:13:55,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-08 08:13:55,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-08 08:13:55,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-08 08:13:55,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-08 08:13:55,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-08 08:13:55,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-08 08:13:55,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-08 08:13:55,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-08 08:13:55,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-08 08:13:55,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-08 08:13:55,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-08 08:13:55,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-08 08:13:55,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-08 08:13:55,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-08 08:13:55,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-08 08:13:55,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-08 08:13:55,965 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08 08:13:55,965 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-08 08:13:55,965 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-08 08:13:55,966 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-08T08:13:56,165 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986 2024-12-08 08:13:56,168 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-08 08:13:56,168 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08T08:13:56,176 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-08T08:13:56,194 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-08T08:13:56,213 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/cluster_7f2d8a74-a01c-b592-2655-65641f2d4b95, deleteOnExit=true 2024-12-08T08:13:56,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-08T08:13:56,215 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/test.cache.data in system properties and HBase conf 2024-12-08T08:13:56,216 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/hadoop.tmp.dir in system properties and HBase conf 2024-12-08T08:13:56,216 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/hadoop.log.dir in system properties and HBase conf 2024-12-08T08:13:56,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-08T08:13:56,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-08T08:13:56,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-08T08:13:56,315 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-08T08:13:56,415 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T08:13:56,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-08T08:13:56,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-08T08:13:56,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-08T08:13:56,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T08:13:56,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-08T08:13:56,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-08T08:13:56,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T08:13:56,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T08:13:56,424 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-08T08:13:56,424 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/nfs.dump.dir in system properties and HBase conf 2024-12-08T08:13:56,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/java.io.tmpdir in system properties and HBase conf 2024-12-08T08:13:56,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T08:13:56,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-08T08:13:56,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-08T08:13:57,803 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-08T08:13:57,866 INFO [Time-limited test {}] log.Log(170): Logging initialized @2661ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-08T08:13:57,932 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T08:13:57,992 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T08:13:58,010 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T08:13:58,010 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T08:13:58,012 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-08T08:13:58,023 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T08:13:58,026 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/hadoop.log.dir/,AVAILABLE} 2024-12-08T08:13:58,027 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T08:13:58,193 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/java.io.tmpdir/jetty-localhost-39909-hadoop-hdfs-3_4_1-tests_jar-_-any-12022666635803619460/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T08:13:58,198 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:39909} 2024-12-08T08:13:58,199 INFO [Time-limited test {}] server.Server(415): Started @2994ms 2024-12-08T08:13:58,992 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T08:13:59,000 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T08:13:59,001 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T08:13:59,001 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T08:13:59,001 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-08T08:13:59,002 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/hadoop.log.dir/,AVAILABLE} 2024-12-08T08:13:59,002 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T08:13:59,114 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/java.io.tmpdir/jetty-localhost-45729-hadoop-hdfs-3_4_1-tests_jar-_-any-4186979051184938131/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T08:13:59,115 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:45729} 2024-12-08T08:13:59,115 INFO [Time-limited test {}] server.Server(415): Started @3910ms 2024-12-08T08:13:59,171 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T08:14:00,245 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/cluster_7f2d8a74-a01c-b592-2655-65641f2d4b95/data/data2/current/BP-729112643-172.17.0.3-1733645636995/current, will proceed with Du for space computation calculation, 2024-12-08T08:14:00,245 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/cluster_7f2d8a74-a01c-b592-2655-65641f2d4b95/data/data1/current/BP-729112643-172.17.0.3-1733645636995/current, will proceed with Du for space computation calculation, 2024-12-08T08:14:00,270 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T08:14:00,316 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6834e83a2f9b7faf with lease ID 0x4f1f537eb5631ffb: Processing first storage report for DS-47631fe3-923b-455f-982c-de9a848f2ee7 from datanode DatanodeRegistration(127.0.0.1:41801, datanodeUuid=760ec621-ddc7-40f2-983b-b5c7ee81a470, infoPort=35703, infoSecurePort=0, ipcPort=32781, storageInfo=lv=-57;cid=testClusterID;nsid=483409731;c=1733645636995) 2024-12-08T08:14:00,317 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6834e83a2f9b7faf with lease ID 0x4f1f537eb5631ffb: from storage DS-47631fe3-923b-455f-982c-de9a848f2ee7 node DatanodeRegistration(127.0.0.1:41801, datanodeUuid=760ec621-ddc7-40f2-983b-b5c7ee81a470, infoPort=35703, infoSecurePort=0, ipcPort=32781, storageInfo=lv=-57;cid=testClusterID;nsid=483409731;c=1733645636995), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-08T08:14:00,318 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6834e83a2f9b7faf with lease ID 0x4f1f537eb5631ffb: Processing first storage report for DS-1abe704d-b1a4-498f-9c32-7f35e454838f from datanode DatanodeRegistration(127.0.0.1:41801, datanodeUuid=760ec621-ddc7-40f2-983b-b5c7ee81a470, infoPort=35703, infoSecurePort=0, ipcPort=32781, storageInfo=lv=-57;cid=testClusterID;nsid=483409731;c=1733645636995) 2024-12-08T08:14:00,318 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6834e83a2f9b7faf with lease ID 0x4f1f537eb5631ffb: from storage DS-1abe704d-b1a4-498f-9c32-7f35e454838f node DatanodeRegistration(127.0.0.1:41801, datanodeUuid=760ec621-ddc7-40f2-983b-b5c7ee81a470, infoPort=35703, infoSecurePort=0, ipcPort=32781, storageInfo=lv=-57;cid=testClusterID;nsid=483409731;c=1733645636995), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T08:14:00,409 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986 2024-12-08T08:14:00,474 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/cluster_7f2d8a74-a01c-b592-2655-65641f2d4b95/zookeeper_0, clientPort=59807, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/cluster_7f2d8a74-a01c-b592-2655-65641f2d4b95/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/cluster_7f2d8a74-a01c-b592-2655-65641f2d4b95/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T08:14:00,485 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59807 2024-12-08T08:14:00,493 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:14:00,496 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:14:00,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741825_1001 (size=7) 2024-12-08T08:14:01,091 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b with version=8 2024-12-08T08:14:01,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/hbase-staging 2024-12-08T08:14:01,168 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-08T08:14:01,421 INFO [Time-limited test {}] client.ConnectionUtils(128): master/422e41b75614:0 server-side Connection retries=45 2024-12-08T08:14:01,429 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T08:14:01,429 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T08:14:01,433 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T08:14:01,433 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T08:14:01,434 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T08:14:01,548 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T08:14:01,600 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-08T08:14:01,608 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-08T08:14:01,612 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T08:14:01,634 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4789 (auto-detected) 2024-12-08T08:14:01,635 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-08T08:14:01,651 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44803 2024-12-08T08:14:01,670 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44803 connecting to ZooKeeper ensemble=127.0.0.1:59807 2024-12-08T08:14:01,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:448030x0, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T08:14:01,791 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44803-0x100047d29d00000 connected 2024-12-08T08:14:01,885 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:14:01,888 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:14:01,898 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T08:14:01,902 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b, hbase.cluster.distributed=false 2024-12-08T08:14:01,921 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T08:14:01,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44803 2024-12-08T08:14:01,926 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44803 2024-12-08T08:14:01,926 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44803 2024-12-08T08:14:01,926 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44803 2024-12-08T08:14:01,927 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44803 2024-12-08T08:14:02,010 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/422e41b75614:0 server-side Connection retries=45 2024-12-08T08:14:02,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T08:14:02,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T08:14:02,012 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T08:14:02,012 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T08:14:02,012 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T08:14:02,014 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T08:14:02,016 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T08:14:02,017 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38619 2024-12-08T08:14:02,018 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38619 connecting to ZooKeeper ensemble=127.0.0.1:59807 2024-12-08T08:14:02,019 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:14:02,022 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:14:02,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386190x0, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T08:14:02,039 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:386190x0, quorum=127.0.0.1:59807, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T08:14:02,039 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38619-0x100047d29d00001 connected 2024-12-08T08:14:02,042 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T08:14:02,049 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T08:14:02,051 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T08:14:02,055 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T08:14:02,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38619 2024-12-08T08:14:02,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38619 2024-12-08T08:14:02,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38619 2024-12-08T08:14:02,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38619 2024-12-08T08:14:02,058 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38619 2024-12-08T08:14:02,070 DEBUG [M:0;422e41b75614:44803 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;422e41b75614:44803 2024-12-08T08:14:02,071 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/422e41b75614,44803,1733645641278 2024-12-08T08:14:02,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T08:14:02,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T08:14:02,084 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/422e41b75614,44803,1733645641278 2024-12-08T08:14:02,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T08:14:02,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:14:02,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:14:02,115 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T08:14:02,116 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/422e41b75614,44803,1733645641278 from backup master directory 2024-12-08T08:14:02,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T08:14:02,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/422e41b75614,44803,1733645641278 2024-12-08T08:14:02,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T08:14:02,125 WARN [master/422e41b75614:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T08:14:02,125 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=422e41b75614,44803,1733645641278 2024-12-08T08:14:02,127 INFO [master/422e41b75614:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-08T08:14:02,129 INFO [master/422e41b75614:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-08T08:14:02,178 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/hbase.id] with ID: 7b8bced7-bfb5-4692-8aa0-6ad00e130e42 2024-12-08T08:14:02,179 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/.tmp/hbase.id 2024-12-08T08:14:02,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741826_1002 (size=42) 2024-12-08T08:14:02,592 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/.tmp/hbase.id]:[hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/hbase.id] 2024-12-08T08:14:02,638 INFO [master/422e41b75614:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:14:02,643 INFO [master/422e41b75614:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T08:14:02,659 INFO [master/422e41b75614:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-08T08:14:02,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:14:02,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:14:02,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741827_1003 (size=196) 2024-12-08T08:14:03,143 INFO [master/422e41b75614:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:14:03,145 INFO [master/422e41b75614:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T08:14:03,158 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:03,161 INFO [master/422e41b75614:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T08:14:03,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741828_1004 (size=1189) 2024-12-08T08:14:03,609 INFO [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store 2024-12-08T08:14:03,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741829_1005 (size=34) 2024-12-08T08:14:04,037 INFO [master/422e41b75614:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-08T08:14:04,040 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:14:04,041 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T08:14:04,041 INFO [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:14:04,041 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:14:04,042 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T08:14:04,042 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:14:04,043 INFO [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:14:04,044 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733645644041Disabling compacts and flushes for region at 1733645644041Disabling writes for close at 1733645644042 (+1 ms)Writing region close event to WAL at 1733645644043 (+1 ms)Closed at 1733645644043 2024-12-08T08:14:04,046 WARN [master/422e41b75614:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/.initializing 2024-12-08T08:14:04,046 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/WALs/422e41b75614,44803,1733645641278 2024-12-08T08:14:04,053 INFO [master/422e41b75614:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T08:14:04,066 INFO [master/422e41b75614:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=422e41b75614%2C44803%2C1733645641278, suffix=, logDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/WALs/422e41b75614,44803,1733645641278, archiveDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/oldWALs, maxLogs=10 2024-12-08T08:14:04,085 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/WALs/422e41b75614,44803,1733645641278/422e41b75614%2C44803%2C1733645641278.1733645644070, exclude list is [], retry=0 2024-12-08T08:14:04,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41801,DS-47631fe3-923b-455f-982c-de9a848f2ee7,DISK] 2024-12-08T08:14:04,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-08T08:14:04,131 INFO [master/422e41b75614:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/WALs/422e41b75614,44803,1733645641278/422e41b75614%2C44803%2C1733645641278.1733645644070 2024-12-08T08:14:04,132 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35703:35703)] 2024-12-08T08:14:04,132 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:14:04,133 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:14:04,135 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:14:04,136 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:14:04,167 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:14:04,188 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T08:14:04,191 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:04,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:14:04,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:14:04,196 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T08:14:04,196 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:04,197 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:14:04,198 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:14:04,200 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T08:14:04,200 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:04,201 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:14:04,201 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:14:04,204 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T08:14:04,204 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:04,205 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:14:04,205 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:14:04,208 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:14:04,209 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:14:04,214 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:14:04,214 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:14:04,217 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T08:14:04,221 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:14:04,226 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:14:04,227 INFO [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62212812, jitterRate=-0.0729568600654602}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T08:14:04,232 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733645644146Initializing all the Stores at 1733645644148 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733645644148Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645644149 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645644149Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645644149Cleaning up temporary data from old regions at 1733645644214 (+65 ms)Region opened successfully at 1733645644232 (+18 ms) 2024-12-08T08:14:04,233 INFO [master/422e41b75614:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T08:14:04,262 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e10692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=422e41b75614/172.17.0.3:0 2024-12-08T08:14:04,294 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T08:14:04,303 INFO [master/422e41b75614:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T08:14:04,303 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T08:14:04,306 INFO [master/422e41b75614:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T08:14:04,307 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-08T08:14:04,311 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-08T08:14:04,311 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T08:14:04,339 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T08:14:04,347 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T08:14:04,409 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T08:14:04,413 INFO [master/422e41b75614:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T08:14:04,415 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T08:14:04,427 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T08:14:04,432 INFO [master/422e41b75614:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T08:14:04,438 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T08:14:04,450 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T08:14:04,453 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T08:14:04,461 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T08:14:04,482 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T08:14:04,492 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T08:14:04,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T08:14:04,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T08:14:04,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:14:04,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:14:04,507 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=422e41b75614,44803,1733645641278, sessionid=0x100047d29d00000, setting cluster-up flag (Was=false) 2024-12-08T08:14:04,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:14:04,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:14:04,567 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T08:14:04,570 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=422e41b75614,44803,1733645641278 2024-12-08T08:14:04,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:14:04,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:14:04,629 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T08:14:04,632 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=422e41b75614,44803,1733645641278 2024-12-08T08:14:04,640 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T08:14:04,664 INFO [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(746): ClusterId : 7b8bced7-bfb5-4692-8aa0-6ad00e130e42 2024-12-08T08:14:04,666 DEBUG [RS:0;422e41b75614:38619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T08:14:04,682 DEBUG [RS:0;422e41b75614:38619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T08:14:04,682 DEBUG [RS:0;422e41b75614:38619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T08:14:04,694 DEBUG [RS:0;422e41b75614:38619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T08:14:04,695 DEBUG [RS:0;422e41b75614:38619 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db856c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=422e41b75614/172.17.0.3:0 2024-12-08T08:14:04,703 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T08:14:04,710 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;422e41b75614:38619 2024-12-08T08:14:04,711 INFO [master/422e41b75614:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T08:14:04,713 INFO [RS:0;422e41b75614:38619 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T08:14:04,713 INFO [RS:0;422e41b75614:38619 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T08:14:04,713 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T08:14:04,716 INFO [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(2659): reportForDuty to master=422e41b75614,44803,1733645641278 with port=38619, startcode=1733645641981 2024-12-08T08:14:04,720 INFO [master/422e41b75614:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T08:14:04,725 DEBUG [RS:0;422e41b75614:38619 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T08:14:04,726 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 422e41b75614,44803,1733645641278 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T08:14:04,734 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/422e41b75614:0, corePoolSize=5, maxPoolSize=5 2024-12-08T08:14:04,734 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/422e41b75614:0, corePoolSize=5, maxPoolSize=5 2024-12-08T08:14:04,734 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/422e41b75614:0, corePoolSize=5, maxPoolSize=5 2024-12-08T08:14:04,735 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/422e41b75614:0, corePoolSize=5, maxPoolSize=5 2024-12-08T08:14:04,735 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/422e41b75614:0, corePoolSize=10, maxPoolSize=10 2024-12-08T08:14:04,735 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:14:04,735 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/422e41b75614:0, corePoolSize=2, maxPoolSize=2 2024-12-08T08:14:04,735 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:14:04,737 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733645674737 2024-12-08T08:14:04,739 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T08:14:04,740 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T08:14:04,740 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T08:14:04,741 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T08:14:04,744 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T08:14:04,745 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T08:14:04,745 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T08:14:04,745 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T08:14:04,748 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:04,748 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T08:14:04,747 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,751 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T08:14:04,752 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T08:14:04,753 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T08:14:04,755 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T08:14:04,755 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T08:14:04,760 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/422e41b75614:0:becomeActiveMaster-HFileCleaner.large.0-1733645644756,5,FailOnTimeoutGroup] 2024-12-08T08:14:04,761 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/422e41b75614:0:becomeActiveMaster-HFileCleaner.small.0-1733645644761,5,FailOnTimeoutGroup] 2024-12-08T08:14:04,762 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,762 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T08:14:04,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741831_1007 (size=1321) 2024-12-08T08:14:04,763 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46429, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T08:14:04,763 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,764 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,768 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44803 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 422e41b75614,38619,1733645641981 2024-12-08T08:14:04,770 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44803 {}] master.ServerManager(517): Registering regionserver=422e41b75614,38619,1733645641981 2024-12-08T08:14:04,783 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b 2024-12-08T08:14:04,783 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43591 2024-12-08T08:14:04,783 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T08:14:04,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T08:14:04,798 DEBUG [RS:0;422e41b75614:38619 {}] zookeeper.ZKUtil(111): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/422e41b75614,38619,1733645641981 2024-12-08T08:14:04,799 WARN [RS:0;422e41b75614:38619 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T08:14:04,799 INFO [RS:0;422e41b75614:38619 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T08:14:04,799 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/WALs/422e41b75614,38619,1733645641981 2024-12-08T08:14:04,801 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [422e41b75614,38619,1733645641981] 2024-12-08T08:14:04,825 INFO [RS:0;422e41b75614:38619 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T08:14:04,837 INFO [RS:0;422e41b75614:38619 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T08:14:04,842 INFO [RS:0;422e41b75614:38619 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T08:14:04,842 INFO [RS:0;422e41b75614:38619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,843 INFO [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T08:14:04,849 INFO [RS:0;422e41b75614:38619 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T08:14:04,850 INFO [RS:0;422e41b75614:38619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,850 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:14:04,850 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:14:04,850 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:14:04,850 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:14:04,851 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:14:04,851 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/422e41b75614:0, corePoolSize=2, maxPoolSize=2 2024-12-08T08:14:04,851 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:14:04,851 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:14:04,851 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:14:04,851 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:14:04,851 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:14:04,852 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:14:04,852 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/422e41b75614:0, corePoolSize=3, maxPoolSize=3 2024-12-08T08:14:04,852 DEBUG [RS:0;422e41b75614:38619 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0, corePoolSize=3, maxPoolSize=3 2024-12-08T08:14:04,853 INFO [RS:0;422e41b75614:38619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,854 INFO [RS:0;422e41b75614:38619 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,854 INFO [RS:0;422e41b75614:38619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,854 INFO [RS:0;422e41b75614:38619 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,854 INFO [RS:0;422e41b75614:38619 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,854 INFO [RS:0;422e41b75614:38619 {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,38619,1733645641981-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T08:14:04,871 INFO [RS:0;422e41b75614:38619 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T08:14:04,872 INFO [RS:0;422e41b75614:38619 {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,38619,1733645641981-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,873 INFO [RS:0;422e41b75614:38619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,873 INFO [RS:0;422e41b75614:38619 {}] regionserver.Replication(171): 422e41b75614,38619,1733645641981 started 2024-12-08T08:14:04,890 INFO [RS:0;422e41b75614:38619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:04,890 INFO [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1482): Serving as 422e41b75614,38619,1733645641981, RpcServer on 422e41b75614/172.17.0.3:38619, sessionid=0x100047d29d00001 2024-12-08T08:14:04,891 DEBUG [RS:0;422e41b75614:38619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T08:14:04,891 DEBUG [RS:0;422e41b75614:38619 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 422e41b75614,38619,1733645641981 2024-12-08T08:14:04,892 DEBUG [RS:0;422e41b75614:38619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '422e41b75614,38619,1733645641981' 2024-12-08T08:14:04,892 DEBUG [RS:0;422e41b75614:38619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T08:14:04,893 DEBUG [RS:0;422e41b75614:38619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T08:14:04,893 DEBUG [RS:0;422e41b75614:38619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T08:14:04,894 DEBUG [RS:0;422e41b75614:38619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T08:14:04,894 DEBUG [RS:0;422e41b75614:38619 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 422e41b75614,38619,1733645641981 2024-12-08T08:14:04,894 DEBUG [RS:0;422e41b75614:38619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '422e41b75614,38619,1733645641981' 2024-12-08T08:14:04,894 DEBUG [RS:0;422e41b75614:38619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T08:14:04,895 DEBUG [RS:0;422e41b75614:38619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T08:14:04,895 DEBUG [RS:0;422e41b75614:38619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T08:14:04,895 INFO [RS:0;422e41b75614:38619 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T08:14:04,896 INFO [RS:0;422e41b75614:38619 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T08:14:05,009 INFO [RS:0;422e41b75614:38619 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T08:14:05,012 INFO [RS:0;422e41b75614:38619 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=422e41b75614%2C38619%2C1733645641981, suffix=, logDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/WALs/422e41b75614,38619,1733645641981, archiveDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/oldWALs, maxLogs=32 2024-12-08T08:14:05,024 DEBUG [RS:0;422e41b75614:38619 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/WALs/422e41b75614,38619,1733645641981/422e41b75614%2C38619%2C1733645641981.1733645645013, exclude list is [], retry=0 2024-12-08T08:14:05,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41801,DS-47631fe3-923b-455f-982c-de9a848f2ee7,DISK] 2024-12-08T08:14:05,032 INFO [RS:0;422e41b75614:38619 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/WALs/422e41b75614,38619,1733645641981/422e41b75614%2C38619%2C1733645641981.1733645645013 2024-12-08T08:14:05,032 DEBUG [RS:0;422e41b75614:38619 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35703:35703)] 2024-12-08T08:14:05,166 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T08:14:05,167 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b 2024-12-08T08:14:05,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741833_1009 (size=32) 2024-12-08T08:14:05,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:14:05,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T08:14:05,592 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T08:14:05,592 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:05,593 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:14:05,593 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T08:14:05,596 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T08:14:05,596 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:05,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:14:05,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T08:14:05,599 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T08:14:05,599 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:05,600 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:14:05,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T08:14:05,603 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T08:14:05,603 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:05,604 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:14:05,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T08:14:05,605 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740 2024-12-08T08:14:05,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740 2024-12-08T08:14:05,608 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T08:14:05,608 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T08:14:05,609 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T08:14:05,611 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T08:14:05,614 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:14:05,615 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66832130, jitterRate=-0.004123657941818237}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T08:14:05,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733645645583Initializing all the Stores at 1733645645586 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733645645586Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733645645587 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645645587Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733645645587Cleaning up temporary data from old regions at 1733645645608 (+21 ms)Region opened successfully at 1733645645617 (+9 ms) 2024-12-08T08:14:05,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T08:14:05,617 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T08:14:05,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T08:14:05,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T08:14:05,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T08:14:05,618 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T08:14:05,619 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733645645617Disabling compacts and flushes for region at 1733645645617Disabling writes for close at 1733645645617Writing region close event to WAL at 1733645645618 (+1 ms)Closed at 1733645645618 2024-12-08T08:14:05,621 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T08:14:05,622 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T08:14:05,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T08:14:05,634 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T08:14:05,637 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T08:14:05,793 DEBUG [422e41b75614:44803 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-08T08:14:05,807 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:14:05,813 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 422e41b75614,38619,1733645641981, state=OPENING 2024-12-08T08:14:05,872 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T08:14:05,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:14:05,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:14:05,885 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T08:14:05,885 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T08:14:05,889 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T08:14:05,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:14:06,070 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T08:14:06,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50691, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T08:14:06,083 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T08:14:06,083 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T08:14:06,084 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T08:14:06,087 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=422e41b75614%2C38619%2C1733645641981.meta, suffix=.meta, logDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/WALs/422e41b75614,38619,1733645641981, archiveDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/oldWALs, maxLogs=32 2024-12-08T08:14:06,100 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/WALs/422e41b75614,38619,1733645641981/422e41b75614%2C38619%2C1733645641981.meta.1733645646088.meta, exclude list is [], retry=0 2024-12-08T08:14:06,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41801,DS-47631fe3-923b-455f-982c-de9a848f2ee7,DISK] 2024-12-08T08:14:06,106 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/WALs/422e41b75614,38619,1733645641981/422e41b75614%2C38619%2C1733645641981.meta.1733645646088.meta 2024-12-08T08:14:06,107 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35703:35703)] 2024-12-08T08:14:06,107 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:14:06,109 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T08:14:06,112 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T08:14:06,117 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T08:14:06,122 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T08:14:06,123 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:14:06,123 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T08:14:06,123 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T08:14:06,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T08:14:06,128 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T08:14:06,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:06,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:14:06,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T08:14:06,131 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T08:14:06,131 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:06,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:14:06,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T08:14:06,134 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T08:14:06,134 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:06,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:14:06,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T08:14:06,136 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T08:14:06,136 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:06,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:14:06,137 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T08:14:06,139 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740 2024-12-08T08:14:06,141 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740 2024-12-08T08:14:06,143 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T08:14:06,143 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T08:14:06,144 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T08:14:06,147 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T08:14:06,150 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62145369, jitterRate=-0.07396183907985687}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T08:14:06,150 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T08:14:06,152 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733645646124Writing region info on filesystem at 1733645646124Initializing all the Stores at 1733645646126 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733645646126Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733645646126Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645646126Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733645646126Cleaning up temporary data from old regions at 1733645646143 (+17 ms)Running coprocessor post-open hooks at 1733645646150 (+7 ms)Region opened successfully at 1733645646152 (+2 ms) 2024-12-08T08:14:06,160 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733645646062 2024-12-08T08:14:06,173 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T08:14:06,173 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T08:14:06,175 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:14:06,177 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 422e41b75614,38619,1733645641981, state=OPEN 2024-12-08T08:14:06,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T08:14:06,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T08:14:06,216 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T08:14:06,216 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T08:14:06,216 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:06,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T08:14:06,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=422e41b75614,38619,1733645641981 in 325 msec 2024-12-08T08:14:06,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T08:14:06,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 597 msec 2024-12-08T08:14:06,232 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T08:14:06,232 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T08:14:06,250 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:06,251 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:06,269 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:06,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43927, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:06,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6240 sec 2024-12-08T08:14:06,293 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733645646293, completionTime=-1 2024-12-08T08:14:06,296 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-08T08:14:06,297 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T08:14:06,325 INFO [master/422e41b75614:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-08T08:14:06,325 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733645706325 2024-12-08T08:14:06,325 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733645766325 2024-12-08T08:14:06,325 INFO [master/422e41b75614:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-12-08T08:14:06,328 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,44803,1733645641278-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:06,328 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,44803,1733645641278-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:06,329 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,44803,1733645641278-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:06,330 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-422e41b75614:44803, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:06,331 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:06,332 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:06,336 DEBUG [master/422e41b75614:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T08:14:06,355 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.228sec 2024-12-08T08:14:06,356 INFO [master/422e41b75614:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T08:14:06,357 INFO [master/422e41b75614:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T08:14:06,358 INFO [master/422e41b75614:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T08:14:06,358 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T08:14:06,358 INFO [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T08:14:06,359 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,44803,1733645641278-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T08:14:06,360 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,44803,1733645641278-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T08:14:06,367 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T08:14:06,368 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T08:14:06,368 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,44803,1733645641278-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T08:14:06,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:06,375 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-08T08:14:06,375 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-08T08:14:06,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:06,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:06,386 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:06,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:06,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:06,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72cbfc6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:06,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:06,393 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:06,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62702105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:06,397 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:06,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:06,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:06,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:06,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=422e41b75614,44803,1733645641278 2024-12-08T08:14:06,437 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=307, ProcessCount=11, AvailableMemoryMB=7042 2024-12-08T08:14:06,444 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T08:14:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 422e41b75614,44803,1733645641278 2024-12-08T08:14:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@b9f8d24 2024-12-08T08:14:06,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T08:14:06,477 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T08:14:06,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T08:14:06,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:14:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T08:14:06,496 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:14:06,499 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-08T08:14:06,499 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:06,502 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:14:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T08:14:06,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741835_1011 (size=1061) 2024-12-08T08:14:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T08:14:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T08:14:06,922 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b 2024-12-08T08:14:06,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741836_1012 (size=53) 2024-12-08T08:14:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T08:14:07,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:14:07,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6e9f0e742ef7ddcc8f58c1d611d5e17b, disabling compactions & flushes 2024-12-08T08:14:07,336 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:07,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:07,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. after waiting 0 ms 2024-12-08T08:14:07,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:07,337 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:07,338 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: Waiting for close lock at 1733645647336Disabling compacts and flushes for region at 1733645647336Disabling writes for close at 1733645647337 (+1 ms)Writing region close event to WAL at 1733645647337Closed at 1733645647337 2024-12-08T08:14:07,342 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:14:07,348 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733645647342"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733645647342"}]},"ts":"1733645647342"} 2024-12-08T08:14:07,352 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:14:07,354 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:14:07,356 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645647354"}]},"ts":"1733645647354"} 2024-12-08T08:14:07,360 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T08:14:07,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e9f0e742ef7ddcc8f58c1d611d5e17b, ASSIGN}] 2024-12-08T08:14:07,364 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e9f0e742ef7ddcc8f58c1d611d5e17b, ASSIGN 2024-12-08T08:14:07,366 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e9f0e742ef7ddcc8f58c1d611d5e17b, ASSIGN; state=OFFLINE, location=422e41b75614,38619,1733645641981; forceNewPlan=false, retain=false 2024-12-08T08:14:07,517 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6e9f0e742ef7ddcc8f58c1d611d5e17b, regionState=OPENING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:14:07,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e9f0e742ef7ddcc8f58c1d611d5e17b, ASSIGN because future has completed 2024-12-08T08:14:07,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:14:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T08:14:07,690 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:07,690 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:14:07,691 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:07,691 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:14:07,691 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:07,692 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:07,695 INFO [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:07,698 INFO [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:14:07,698 INFO [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e9f0e742ef7ddcc8f58c1d611d5e17b columnFamilyName A 2024-12-08T08:14:07,698 DEBUG [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:07,700 INFO [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] regionserver.HStore(327): Store=6e9f0e742ef7ddcc8f58c1d611d5e17b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:14:07,700 INFO [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:07,702 INFO [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:14:07,702 INFO [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e9f0e742ef7ddcc8f58c1d611d5e17b columnFamilyName B 2024-12-08T08:14:07,702 DEBUG [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:07,703 INFO [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] regionserver.HStore(327): Store=6e9f0e742ef7ddcc8f58c1d611d5e17b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:14:07,703 INFO [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:07,705 INFO [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:14:07,705 INFO [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e9f0e742ef7ddcc8f58c1d611d5e17b columnFamilyName C 2024-12-08T08:14:07,705 DEBUG [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:07,706 INFO [StoreOpener-6e9f0e742ef7ddcc8f58c1d611d5e17b-1 {}] regionserver.HStore(327): Store=6e9f0e742ef7ddcc8f58c1d611d5e17b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:14:07,707 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:07,707 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:07,709 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:07,709 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:07,711 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:07,711 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:07,712 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T08:14:07,715 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:07,718 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:14:07,720 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6e9f0e742ef7ddcc8f58c1d611d5e17b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61316782, jitterRate=-0.08630874752998352}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T08:14:07,720 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:07,721 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: Running coprocessor pre-open hook at 1733645647692Writing region info on filesystem at 1733645647692Initializing all the Stores at 1733645647694 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645647694Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645647695 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645647695Cleaning up temporary data from old regions at 1733645647711 (+16 ms)Running coprocessor post-open hooks at 1733645647720 (+9 ms)Region opened successfully at 1733645647721 (+1 ms) 2024-12-08T08:14:07,723 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., pid=6, masterSystemTime=1733645647679 2024-12-08T08:14:07,727 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:07,727 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:07,728 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6e9f0e742ef7ddcc8f58c1d611d5e17b, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:14:07,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:14:07,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T08:14:07,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 in 211 msec 2024-12-08T08:14:07,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T08:14:07,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e9f0e742ef7ddcc8f58c1d611d5e17b, ASSIGN in 377 msec 2024-12-08T08:14:07,745 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:14:07,746 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645647745"}]},"ts":"1733645647745"} 2024-12-08T08:14:07,749 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T08:14:07,751 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:14:07,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2660 sec 2024-12-08T08:14:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T08:14:08,653 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:08,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:08,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:08,665 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508e5ff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:08,670 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fe15c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,673 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:08,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:08,679 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T08:14:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 422e41b75614,44803,1733645641278 2024-12-08T08:14:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@253960f5 2024-12-08T08:14:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T08:14:08,686 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T08:14:08,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:08,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:08,692 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395f1c25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:08,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:08,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:08,695 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec6ad86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:08,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:08,698 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ac3455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:08,701 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b0e30d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:08,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:08,706 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:08,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:08,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74cae7c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:08,709 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6573bf69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:08,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:08,713 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c0df3b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:08,717 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59441750, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,719 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:08,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:08,720 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:08,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:08,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cdbfa6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:08,723 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@420cbead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:08,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:08,727 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dfb2ab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:08,729 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce3cf61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:08,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:08,732 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa07968, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:08,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:08,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:08,735 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5097bf60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:08,737 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:08,738 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12dac1a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:08,740 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ce25cec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:08,742 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:08,743 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6034ec4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:08,745 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b32f21b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:08,750 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:08,750 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:08,750 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:08,750 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:08,751 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:08,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:08,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:08,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:08,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:08,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:08,758 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:14:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:14:08,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:14:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:14:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:14:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:14:08,768 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:08,773 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:08,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:14:08,777 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:08,781 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:08,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:08,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:14:08,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-08T08:14:08,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:08,788 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:14:08,791 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:08,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T08:14:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:08,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T08:14:08,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:08,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:08,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:08,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:08,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:08,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T08:14:08,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/d13a483eb25b4d9688d51b3625e5ba3e is 50, key is test_row_0/A:col10/1733645648829/Put/seqid=0 2024-12-08T08:14:08,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645708917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:08,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645708917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:08,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645708939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:08,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:08,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645708921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645708940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:08,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741837_1013 (size=12001) 2024-12-08T08:14:08,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T08:14:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:08,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:08,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:08,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:08,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:08,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:08,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645709083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645709084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645709084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645709085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645709088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T08:14:09,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T08:14:09,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:09,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:09,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:09,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645709291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645709292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645709303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645709303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645709303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T08:14:09,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:09,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:09,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:09,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/d13a483eb25b4d9688d51b3625e5ba3e 2024-12-08T08:14:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T08:14:09,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/939c36cd24c9471cb815f91c07bc94be is 50, key is test_row_0/B:col10/1733645648829/Put/seqid=0 2024-12-08T08:14:09,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T08:14:09,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:09,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:09,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:09,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741838_1014 (size=12001) 2024-12-08T08:14:09,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/939c36cd24c9471cb815f91c07bc94be 2024-12-08T08:14:09,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/99a515751188472da396dfce9a45f579 is 50, key is test_row_0/C:col10/1733645648829/Put/seqid=0 2024-12-08T08:14:09,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741839_1015 (size=12001) 2024-12-08T08:14:09,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645709601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645709611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645709612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645709613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645709612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:09,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T08:14:09,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:09,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:09,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:09,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T08:14:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:09,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T08:14:09,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T08:14:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:09,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:09,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/99a515751188472da396dfce9a45f579 2024-12-08T08:14:09,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/d13a483eb25b4d9688d51b3625e5ba3e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d13a483eb25b4d9688d51b3625e5ba3e 2024-12-08T08:14:09,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d13a483eb25b4d9688d51b3625e5ba3e, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T08:14:09,991 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-08T08:14:09,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/939c36cd24c9471cb815f91c07bc94be as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/939c36cd24c9471cb815f91c07bc94be 2024-12-08T08:14:10,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/939c36cd24c9471cb815f91c07bc94be, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T08:14:10,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/99a515751188472da396dfce9a45f579 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/99a515751188472da396dfce9a45f579 2024-12-08T08:14:10,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/99a515751188472da396dfce9a45f579, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T08:14:10,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 1206ms, sequenceid=13, compaction requested=false 2024-12-08T08:14:10,041 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T08:14:10,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:10,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T08:14:10,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:10,093 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T08:14:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:10,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/dfbaf8c57a2040e194ea459dfa1d2007 is 50, key is test_row_0/A:col10/1733645648929/Put/seqid=0 2024-12-08T08:14:10,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:10,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741840_1016 (size=12001) 2024-12-08T08:14:10,139 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/dfbaf8c57a2040e194ea459dfa1d2007 2024-12-08T08:14:10,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/cac01e61271d499d88bcca26b939df90 is 50, key is test_row_0/B:col10/1733645648929/Put/seqid=0 2024-12-08T08:14:10,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645710154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645710161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645710171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645710178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645710186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741841_1017 (size=12001) 2024-12-08T08:14:10,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645710291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645710311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645710311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645710313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645710322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645710521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645710531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645710531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645710532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645710532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,617 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/cac01e61271d499d88bcca26b939df90 2024-12-08T08:14:10,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/6f2754910a3249eebdf2c936ed0e8b01 is 50, key is test_row_0/C:col10/1733645648929/Put/seqid=0 2024-12-08T08:14:10,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741842_1018 (size=12001) 2024-12-08T08:14:10,695 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/6f2754910a3249eebdf2c936ed0e8b01 2024-12-08T08:14:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/dfbaf8c57a2040e194ea459dfa1d2007 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/dfbaf8c57a2040e194ea459dfa1d2007 2024-12-08T08:14:10,737 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/dfbaf8c57a2040e194ea459dfa1d2007, entries=150, sequenceid=33, filesize=11.7 K 2024-12-08T08:14:10,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/cac01e61271d499d88bcca26b939df90 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/cac01e61271d499d88bcca26b939df90 2024-12-08T08:14:10,766 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/cac01e61271d499d88bcca26b939df90, entries=150, sequenceid=33, filesize=11.7 K 2024-12-08T08:14:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/6f2754910a3249eebdf2c936ed0e8b01 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/6f2754910a3249eebdf2c936ed0e8b01 2024-12-08T08:14:10,786 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/6f2754910a3249eebdf2c936ed0e8b01, entries=150, sequenceid=33, filesize=11.7 K 2024-12-08T08:14:10,788 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 695ms, sequenceid=33, compaction requested=false 2024-12-08T08:14:10,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:10,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:10,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-08T08:14:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-08T08:14:10,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T08:14:10,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0000 sec 2024-12-08T08:14:10,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.0220 sec 2024-12-08T08:14:10,820 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T08:14:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:10,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T08:14:10,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:10,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:10,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:10,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:10,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:10,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:10,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/178048a2c21c479aba5f0f2789d7138f is 50, key is test_row_0/A:col10/1733645650834/Put/seqid=0 2024-12-08T08:14:10,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741843_1019 (size=12001) 2024-12-08T08:14:10,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/178048a2c21c479aba5f0f2789d7138f 2024-12-08T08:14:10,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/aeb14fcea3c348689c484194d65241eb is 50, key is test_row_0/B:col10/1733645650834/Put/seqid=0 2024-12-08T08:14:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645710901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645710902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645710912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645710921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645710925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T08:14:10,941 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:10,942 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-08T08:14:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T08:14:10,953 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:10,955 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:10,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:10,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741844_1020 (size=12001) 2024-12-08T08:14:10,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/aeb14fcea3c348689c484194d65241eb 2024-12-08T08:14:10,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/4b890577ad724a01b827595f163f6550 is 50, key is test_row_0/C:col10/1733645650834/Put/seqid=0 2024-12-08T08:14:11,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741845_1021 (size=12001) 2024-12-08T08:14:11,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/4b890577ad724a01b827595f163f6550 2024-12-08T08:14:11,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/178048a2c21c479aba5f0f2789d7138f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/178048a2c21c479aba5f0f2789d7138f 2024-12-08T08:14:11,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645711030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645711031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645711031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645711041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645711042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T08:14:11,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/178048a2c21c479aba5f0f2789d7138f, entries=150, sequenceid=46, filesize=11.7 K 2024-12-08T08:14:11,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/aeb14fcea3c348689c484194d65241eb as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/aeb14fcea3c348689c484194d65241eb 2024-12-08T08:14:11,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/aeb14fcea3c348689c484194d65241eb, entries=150, sequenceid=46, filesize=11.7 K 2024-12-08T08:14:11,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/4b890577ad724a01b827595f163f6550 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/4b890577ad724a01b827595f163f6550 2024-12-08T08:14:11,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/4b890577ad724a01b827595f163f6550, entries=150, sequenceid=46, filesize=11.7 K 2024-12-08T08:14:11,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 269ms, sequenceid=46, compaction requested=true 2024-12-08T08:14:11,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:11,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T08:14:11,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:11,112 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T08:14:11,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:11,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:11,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:11,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:11,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:11,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:11,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:11,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:11,123 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:11,125 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:11,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/c17d66f9ac81410a9d381c4875a0bde0 is 50, key is test_row_0/A:col10/1733645650908/Put/seqid=0 2024-12-08T08:14:11,131 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:11,133 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/B is initiating minor compaction (all files) 2024-12-08T08:14:11,133 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/B in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:11,134 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/939c36cd24c9471cb815f91c07bc94be, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/cac01e61271d499d88bcca26b939df90, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/aeb14fcea3c348689c484194d65241eb] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=35.2 K 2024-12-08T08:14:11,136 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:11,136 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/A is initiating minor compaction (all files) 2024-12-08T08:14:11,136 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 939c36cd24c9471cb815f91c07bc94be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733645648812 2024-12-08T08:14:11,136 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/A in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:11,136 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d13a483eb25b4d9688d51b3625e5ba3e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/dfbaf8c57a2040e194ea459dfa1d2007, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/178048a2c21c479aba5f0f2789d7138f] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=35.2 K 2024-12-08T08:14:11,138 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting cac01e61271d499d88bcca26b939df90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733645648915 2024-12-08T08:14:11,139 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting aeb14fcea3c348689c484194d65241eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733645650164 2024-12-08T08:14:11,139 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting d13a483eb25b4d9688d51b3625e5ba3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733645648812 2024-12-08T08:14:11,140 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfbaf8c57a2040e194ea459dfa1d2007, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733645648915 2024-12-08T08:14:11,142 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 178048a2c21c479aba5f0f2789d7138f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733645650164 2024-12-08T08:14:11,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741846_1022 (size=12001) 2024-12-08T08:14:11,183 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/c17d66f9ac81410a9d381c4875a0bde0 2024-12-08T08:14:11,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/2635cc1541a24e8e947729e00ea8293d is 50, key is test_row_0/B:col10/1733645650908/Put/seqid=0 2024-12-08T08:14:11,226 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#A#compaction#10 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:11,227 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/232edbcd4bfb4ab4b35b304a2ce0c0f4 is 50, key is test_row_0/A:col10/1733645650834/Put/seqid=0 2024-12-08T08:14:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T08:14:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:11,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:11,283 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#B#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:11,284 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/5e2d75b9ddbc4fd496e160f95b6f8c11 is 50, key is test_row_0/B:col10/1733645650834/Put/seqid=0 2024-12-08T08:14:11,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741847_1023 (size=12001) 2024-12-08T08:14:11,292 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/2635cc1541a24e8e947729e00ea8293d 2024-12-08T08:14:11,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741848_1024 (size=12104) 2024-12-08T08:14:11,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645711288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645711299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645711303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645711308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645711310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/2306e80e886a4ad9af0e7d019f93c8bb is 50, key is test_row_0/C:col10/1733645650908/Put/seqid=0 2024-12-08T08:14:11,330 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/232edbcd4bfb4ab4b35b304a2ce0c0f4 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/232edbcd4bfb4ab4b35b304a2ce0c0f4 2024-12-08T08:14:11,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741849_1025 (size=12104) 2024-12-08T08:14:11,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741850_1026 (size=12001) 2024-12-08T08:14:11,356 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/2306e80e886a4ad9af0e7d019f93c8bb 2024-12-08T08:14:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/c17d66f9ac81410a9d381c4875a0bde0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/c17d66f9ac81410a9d381c4875a0bde0 2024-12-08T08:14:11,378 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/A of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 232edbcd4bfb4ab4b35b304a2ce0c0f4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:11,378 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:11,384 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/A, priority=13, startTime=1733645651109; duration=0sec 2024-12-08T08:14:11,384 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:11,386 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:A 2024-12-08T08:14:11,386 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:11,391 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:11,391 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/C is initiating minor compaction (all files) 2024-12-08T08:14:11,391 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/C in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:11,391 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/99a515751188472da396dfce9a45f579, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/6f2754910a3249eebdf2c936ed0e8b01, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/4b890577ad724a01b827595f163f6550] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=35.2 K 2024-12-08T08:14:11,392 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99a515751188472da396dfce9a45f579, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733645648812 2024-12-08T08:14:11,392 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/c17d66f9ac81410a9d381c4875a0bde0, entries=150, sequenceid=66, filesize=11.7 K 2024-12-08T08:14:11,394 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f2754910a3249eebdf2c936ed0e8b01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733645648915 2024-12-08T08:14:11,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/2635cc1541a24e8e947729e00ea8293d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/2635cc1541a24e8e947729e00ea8293d 2024-12-08T08:14:11,397 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b890577ad724a01b827595f163f6550, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733645650164 2024-12-08T08:14:11,416 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/2635cc1541a24e8e947729e00ea8293d, entries=150, sequenceid=66, filesize=11.7 K 2024-12-08T08:14:11,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/2306e80e886a4ad9af0e7d019f93c8bb as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/2306e80e886a4ad9af0e7d019f93c8bb 2024-12-08T08:14:11,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645711423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645711423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645711423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645711431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645711431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,444 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/2306e80e886a4ad9af0e7d019f93c8bb, entries=150, sequenceid=66, filesize=11.7 K 2024-12-08T08:14:11,449 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 336ms, sequenceid=66, compaction requested=false 2024-12-08T08:14:11,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:11,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-08T08:14:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-08T08:14:11,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-08T08:14:11,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 503 msec 2024-12-08T08:14:11,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 525 msec 2024-12-08T08:14:11,477 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#C#compaction#14 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:11,477 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/45c50450d9f64529b92104469569e147 is 50, key is test_row_0/C:col10/1733645650834/Put/seqid=0 2024-12-08T08:14:11,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741851_1027 (size=12104) 2024-12-08T08:14:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T08:14:11,571 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:11,572 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-08T08:14:11,577 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T08:14:11,578 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:11,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:11,633 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T08:14:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:11,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T08:14:11,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:11,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:11,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:11,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:11,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:11,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:11,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/1541c999376c488a812f6d51df67d404 is 50, key is test_row_0/A:col10/1733645651632/Put/seqid=0 2024-12-08T08:14:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T08:14:11,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741852_1028 (size=12001) 2024-12-08T08:14:11,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/1541c999376c488a812f6d51df67d404 2024-12-08T08:14:11,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645711685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645711688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645711689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645711699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645711700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/777f481474e64c64948f73012879ef51 is 50, key is test_row_0/B:col10/1733645651632/Put/seqid=0 2024-12-08T08:14:11,723 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-08T08:14:11,723 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-08T08:14:11,726 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T08:14:11,726 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-08T08:14:11,726 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-08T08:14:11,726 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-08T08:14:11,728 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T08:14:11,728 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-08T08:14:11,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T08:14:11,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:11,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:11,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:11,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:11,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:11,749 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/5e2d75b9ddbc4fd496e160f95b6f8c11 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/5e2d75b9ddbc4fd496e160f95b6f8c11 2024-12-08T08:14:11,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741853_1029 (size=12001) 2024-12-08T08:14:11,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/777f481474e64c64948f73012879ef51 2024-12-08T08:14:11,767 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/B of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 5e2d75b9ddbc4fd496e160f95b6f8c11(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:11,767 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:11,767 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/B, priority=13, startTime=1733645651121; duration=0sec 2024-12-08T08:14:11,767 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:11,767 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:B 2024-12-08T08:14:11,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/952a1692537a427dbb63e985cd9d45d2 is 50, key is test_row_0/C:col10/1733645651632/Put/seqid=0 2024-12-08T08:14:11,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741854_1030 (size=12001) 2024-12-08T08:14:11,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/952a1692537a427dbb63e985cd9d45d2 2024-12-08T08:14:11,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645711811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/1541c999376c488a812f6d51df67d404 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1541c999376c488a812f6d51df67d404 2024-12-08T08:14:11,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645711811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645711812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645711821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645711821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:11,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1541c999376c488a812f6d51df67d404, entries=150, sequenceid=80, filesize=11.7 K 2024-12-08T08:14:11,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/777f481474e64c64948f73012879ef51 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/777f481474e64c64948f73012879ef51 2024-12-08T08:14:11,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/777f481474e64c64948f73012879ef51, entries=150, sequenceid=80, filesize=11.7 K 2024-12-08T08:14:11,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/952a1692537a427dbb63e985cd9d45d2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/952a1692537a427dbb63e985cd9d45d2 2024-12-08T08:14:11,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/952a1692537a427dbb63e985cd9d45d2, entries=150, sequenceid=80, filesize=11.7 K 2024-12-08T08:14:11,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 230ms, sequenceid=80, compaction requested=true 2024-12-08T08:14:11,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:11,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:A, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:11,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:11,870 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:11,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:B, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:11,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:11,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:11,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T08:14:11,872 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:11,872 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/A is initiating minor compaction (all files) 2024-12-08T08:14:11,872 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/A in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:11,872 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/232edbcd4bfb4ab4b35b304a2ce0c0f4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/c17d66f9ac81410a9d381c4875a0bde0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1541c999376c488a812f6d51df67d404] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=35.3 K 2024-12-08T08:14:11,873 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 232edbcd4bfb4ab4b35b304a2ce0c0f4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733645650164 2024-12-08T08:14:11,873 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting c17d66f9ac81410a9d381c4875a0bde0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733645650907 2024-12-08T08:14:11,875 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 1541c999376c488a812f6d51df67d404, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733645651301 2024-12-08T08:14:11,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T08:14:11,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:11,890 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T08:14:11,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T08:14:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:11,905 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/45c50450d9f64529b92104469569e147 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/45c50450d9f64529b92104469569e147 2024-12-08T08:14:11,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/c7381e3f767d4bb9a22782c48d449978 is 50, key is test_row_0/A:col10/1733645651695/Put/seqid=0 2024-12-08T08:14:11,910 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#A#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:11,911 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/25c12ae2fdf247ea8486a2695453bf9c is 50, key is test_row_0/A:col10/1733645651632/Put/seqid=0 2024-12-08T08:14:11,920 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/C of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 45c50450d9f64529b92104469569e147(size=11.8 K), total size for store is 35.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:11,920 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:11,920 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/C, priority=13, startTime=1733645651121; duration=0sec 2024-12-08T08:14:11,920 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T08:14:11,920 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:C 2024-12-08T08:14:11,920 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:C 2024-12-08T08:14:11,921 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:11,923 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:11,923 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/B is initiating minor compaction (all files) 2024-12-08T08:14:11,923 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/B in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:11,923 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/5e2d75b9ddbc4fd496e160f95b6f8c11, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/2635cc1541a24e8e947729e00ea8293d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/777f481474e64c64948f73012879ef51] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=35.3 K 2024-12-08T08:14:11,924 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e2d75b9ddbc4fd496e160f95b6f8c11, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733645650164 2024-12-08T08:14:11,925 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2635cc1541a24e8e947729e00ea8293d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733645650907 2024-12-08T08:14:11,926 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 777f481474e64c64948f73012879ef51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733645651301 2024-12-08T08:14:11,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741856_1032 (size=12207) 2024-12-08T08:14:11,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741855_1031 (size=12001) 2024-12-08T08:14:11,962 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/25c12ae2fdf247ea8486a2695453bf9c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/25c12ae2fdf247ea8486a2695453bf9c 2024-12-08T08:14:11,972 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#B#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:11,973 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/a15518b5f499405c80e3c28f4c953b0d is 50, key is test_row_0/B:col10/1733645651632/Put/seqid=0 2024-12-08T08:14:11,980 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/A of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 25c12ae2fdf247ea8486a2695453bf9c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:11,981 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:11,981 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/A, priority=13, startTime=1733645651870; duration=0sec 2024-12-08T08:14:11,981 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:11,981 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:A 2024-12-08T08:14:11,981 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:11,984 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:11,984 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/C is initiating minor compaction (all files) 2024-12-08T08:14:11,984 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/C in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:11,984 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/45c50450d9f64529b92104469569e147, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/2306e80e886a4ad9af0e7d019f93c8bb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/952a1692537a427dbb63e985cd9d45d2] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=35.3 K 2024-12-08T08:14:11,985 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 45c50450d9f64529b92104469569e147, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733645650164 2024-12-08T08:14:11,986 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 2306e80e886a4ad9af0e7d019f93c8bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733645650907 2024-12-08T08:14:11,990 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 952a1692537a427dbb63e985cd9d45d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733645651301 2024-12-08T08:14:12,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741857_1033 (size=12207) 2024-12-08T08:14:12,020 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/a15518b5f499405c80e3c28f4c953b0d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/a15518b5f499405c80e3c28f4c953b0d 2024-12-08T08:14:12,027 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:12,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:12,028 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/361be88b9ff5499482d9a8521accbe7a is 50, key is test_row_0/C:col10/1733645651632/Put/seqid=0 2024-12-08T08:14:12,037 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/B of 6e9f0e742ef7ddcc8f58c1d611d5e17b into a15518b5f499405c80e3c28f4c953b0d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:12,037 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:12,037 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/B, priority=13, startTime=1733645651870; duration=0sec 2024-12-08T08:14:12,040 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:12,040 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:B 2024-12-08T08:14:12,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645712046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645712051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645712053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645712054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645712055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741858_1034 (size=12207) 2024-12-08T08:14:12,080 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/361be88b9ff5499482d9a8521accbe7a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/361be88b9ff5499482d9a8521accbe7a 2024-12-08T08:14:12,093 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/C of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 361be88b9ff5499482d9a8521accbe7a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:12,093 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:12,093 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/C, priority=13, startTime=1733645651870; duration=0sec 2024-12-08T08:14:12,095 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:12,095 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:C 2024-12-08T08:14:12,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645712161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645712171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645712171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645712173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645712171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T08:14:12,361 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/c7381e3f767d4bb9a22782c48d449978 2024-12-08T08:14:12,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645712371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/d01573d1a7b34431a79b721c2bb98329 is 50, key is test_row_0/B:col10/1733645651695/Put/seqid=0 2024-12-08T08:14:12,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645712382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645712382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645712391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645712391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741859_1035 (size=12001) 2024-12-08T08:14:12,432 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/d01573d1a7b34431a79b721c2bb98329 2024-12-08T08:14:12,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/02a394e105014430bb3b2e0dd214da8b is 50, key is test_row_0/C:col10/1733645651695/Put/seqid=0 2024-12-08T08:14:12,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741860_1036 (size=12001) 2024-12-08T08:14:12,485 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/02a394e105014430bb3b2e0dd214da8b 2024-12-08T08:14:12,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/c7381e3f767d4bb9a22782c48d449978 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/c7381e3f767d4bb9a22782c48d449978 2024-12-08T08:14:12,516 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/c7381e3f767d4bb9a22782c48d449978, entries=150, sequenceid=101, filesize=11.7 K 2024-12-08T08:14:12,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/d01573d1a7b34431a79b721c2bb98329 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/d01573d1a7b34431a79b721c2bb98329 2024-12-08T08:14:12,535 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/d01573d1a7b34431a79b721c2bb98329, entries=150, sequenceid=101, filesize=11.7 K 2024-12-08T08:14:12,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/02a394e105014430bb3b2e0dd214da8b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/02a394e105014430bb3b2e0dd214da8b 2024-12-08T08:14:12,567 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/02a394e105014430bb3b2e0dd214da8b, entries=150, sequenceid=101, filesize=11.7 K 2024-12-08T08:14:12,568 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 678ms, sequenceid=101, compaction requested=false 2024-12-08T08:14:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-08T08:14:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-08T08:14:12,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-08T08:14:12,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 992 msec 2024-12-08T08:14:12,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.0060 sec 2024-12-08T08:14:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:12,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T08:14:12,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:12,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:12,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:12,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:12,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:12,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:12,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/18008434e2804efe8216b433f7c83506 is 50, key is test_row_0/A:col10/1733645652050/Put/seqid=0 2024-12-08T08:14:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T08:14:12,711 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:12,712 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-08T08:14:12,716 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:12,718 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:12,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T08:14:12,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741861_1037 (size=14341) 2024-12-08T08:14:12,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/18008434e2804efe8216b433f7c83506 2024-12-08T08:14:12,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645712729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645712731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645712731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645712735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645712742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/0e065b658536402aa70a276873ba6c31 is 50, key is test_row_0/B:col10/1733645652050/Put/seqid=0 2024-12-08T08:14:12,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741862_1038 (size=12001) 2024-12-08T08:14:12,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/0e065b658536402aa70a276873ba6c31 2024-12-08T08:14:12,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/224983b61d0c412dbf98e71c0b80cbb6 is 50, key is test_row_0/C:col10/1733645652050/Put/seqid=0 2024-12-08T08:14:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T08:14:12,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741863_1039 (size=12001) 2024-12-08T08:14:12,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/224983b61d0c412dbf98e71c0b80cbb6 2024-12-08T08:14:12,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/18008434e2804efe8216b433f7c83506 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/18008434e2804efe8216b433f7c83506 2024-12-08T08:14:12,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/18008434e2804efe8216b433f7c83506, entries=200, sequenceid=119, filesize=14.0 K 2024-12-08T08:14:12,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/0e065b658536402aa70a276873ba6c31 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/0e065b658536402aa70a276873ba6c31 2024-12-08T08:14:12,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645712852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645712852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645712852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:12,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645712852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:12,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/0e065b658536402aa70a276873ba6c31, entries=150, sequenceid=119, filesize=11.7 K 2024-12-08T08:14:12,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/224983b61d0c412dbf98e71c0b80cbb6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/224983b61d0c412dbf98e71c0b80cbb6 2024-12-08T08:14:12,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/224983b61d0c412dbf98e71c0b80cbb6, entries=150, sequenceid=119, filesize=11.7 K 2024-12-08T08:14:12,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T08:14:12,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:12,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:12,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:12,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:12,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:12,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 192ms, sequenceid=119, compaction requested=true 2024-12-08T08:14:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:12,878 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:12,878 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:12,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:12,881 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:12,881 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/B is initiating minor compaction (all files) 2024-12-08T08:14:12,881 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/B in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:12,881 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/a15518b5f499405c80e3c28f4c953b0d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/d01573d1a7b34431a79b721c2bb98329, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/0e065b658536402aa70a276873ba6c31] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=35.4 K 2024-12-08T08:14:12,881 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:12,882 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/A is initiating minor compaction (all files) 2024-12-08T08:14:12,882 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/A in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:12,882 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/25c12ae2fdf247ea8486a2695453bf9c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/c7381e3f767d4bb9a22782c48d449978, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/18008434e2804efe8216b433f7c83506] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=37.6 K 2024-12-08T08:14:12,882 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting a15518b5f499405c80e3c28f4c953b0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733645651301 2024-12-08T08:14:12,883 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25c12ae2fdf247ea8486a2695453bf9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733645651301 2024-12-08T08:14:12,884 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting d01573d1a7b34431a79b721c2bb98329, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733645651684 2024-12-08T08:14:12,887 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7381e3f767d4bb9a22782c48d449978, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733645651684 2024-12-08T08:14:12,888 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e065b658536402aa70a276873ba6c31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733645652050 2024-12-08T08:14:12,888 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18008434e2804efe8216b433f7c83506, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733645652049 2024-12-08T08:14:12,913 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#A#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:12,913 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/34a9a4ce644b460387c4328c9314782e is 50, key is test_row_0/A:col10/1733645652050/Put/seqid=0 2024-12-08T08:14:12,917 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:12,918 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/85d8eab801ee4eee861807b922a3e863 is 50, key is test_row_0/B:col10/1733645652050/Put/seqid=0 2024-12-08T08:14:12,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741864_1040 (size=12309) 2024-12-08T08:14:12,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741865_1041 (size=12309) 2024-12-08T08:14:12,964 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/34a9a4ce644b460387c4328c9314782e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/34a9a4ce644b460387c4328c9314782e 2024-12-08T08:14:12,978 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/A of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 34a9a4ce644b460387c4328c9314782e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:12,978 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:12,979 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/A, priority=13, startTime=1733645652878; duration=0sec 2024-12-08T08:14:12,979 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:12,979 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:A 2024-12-08T08:14:12,979 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:12,981 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:12,981 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/C is initiating minor compaction (all files) 2024-12-08T08:14:12,981 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/C in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:12,982 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/361be88b9ff5499482d9a8521accbe7a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/02a394e105014430bb3b2e0dd214da8b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/224983b61d0c412dbf98e71c0b80cbb6] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=35.4 K 2024-12-08T08:14:12,982 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 361be88b9ff5499482d9a8521accbe7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733645651301 2024-12-08T08:14:12,983 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02a394e105014430bb3b2e0dd214da8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733645651684 2024-12-08T08:14:12,984 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 224983b61d0c412dbf98e71c0b80cbb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733645652050 2024-12-08T08:14:13,006 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#C#compaction#29 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:13,007 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/1f17ec22b68c4748b43ac9b28041e37c is 50, key is test_row_0/C:col10/1733645652050/Put/seqid=0 2024-12-08T08:14:13,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741866_1042 (size=12309) 2024-12-08T08:14:13,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T08:14:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:13,033 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T08:14:13,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:13,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:13,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:13,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:13,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:13,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:13,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T08:14:13,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/e4b611d78ebf4506a84dc0791f5eac4c is 50, key is test_row_0/A:col10/1733645652720/Put/seqid=0 2024-12-08T08:14:13,044 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/1f17ec22b68c4748b43ac9b28041e37c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/1f17ec22b68c4748b43ac9b28041e37c 2024-12-08T08:14:13,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741867_1043 (size=12151) 2024-12-08T08:14:13,053 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/e4b611d78ebf4506a84dc0791f5eac4c 2024-12-08T08:14:13,058 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/C of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 1f17ec22b68c4748b43ac9b28041e37c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:13,058 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:13,058 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/C, priority=13, startTime=1733645652878; duration=0sec 2024-12-08T08:14:13,058 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:13,058 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:C 2024-12-08T08:14:13,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:13,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:13,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/34d919a7ab974c68b746e34817306d52 is 50, key is test_row_0/B:col10/1733645652720/Put/seqid=0 2024-12-08T08:14:13,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741868_1044 (size=12151) 2024-12-08T08:14:13,088 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/34d919a7ab974c68b746e34817306d52 2024-12-08T08:14:13,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645713082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645713088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645713092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645713092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/de5068abaead4de0bd7050f7a2d2a037 is 50, key is test_row_0/C:col10/1733645652720/Put/seqid=0 2024-12-08T08:14:13,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741869_1045 (size=12151) 2024-12-08T08:14:13,129 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/de5068abaead4de0bd7050f7a2d2a037 2024-12-08T08:14:13,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/e4b611d78ebf4506a84dc0791f5eac4c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e4b611d78ebf4506a84dc0791f5eac4c 2024-12-08T08:14:13,146 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e4b611d78ebf4506a84dc0791f5eac4c, entries=150, sequenceid=139, filesize=11.9 K 2024-12-08T08:14:13,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/34d919a7ab974c68b746e34817306d52 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/34d919a7ab974c68b746e34817306d52 2024-12-08T08:14:13,157 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/34d919a7ab974c68b746e34817306d52, entries=150, sequenceid=139, filesize=11.9 K 2024-12-08T08:14:13,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/de5068abaead4de0bd7050f7a2d2a037 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/de5068abaead4de0bd7050f7a2d2a037 2024-12-08T08:14:13,172 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/de5068abaead4de0bd7050f7a2d2a037, entries=150, sequenceid=139, filesize=11.9 K 2024-12-08T08:14:13,174 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 142ms, sequenceid=139, compaction requested=false 2024-12-08T08:14:13,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:13,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:13,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-08T08:14:13,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-08T08:14:13,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-08T08:14:13,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 459 msec 2024-12-08T08:14:13,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 471 msec 2024-12-08T08:14:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:13,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T08:14:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:13,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/e213e29eaee14af095dcb85da3efedcf is 50, key is test_row_0/A:col10/1733645653086/Put/seqid=0 2024-12-08T08:14:13,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741870_1046 (size=12151) 2024-12-08T08:14:13,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/e213e29eaee14af095dcb85da3efedcf 2024-12-08T08:14:13,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/45b694078bae44af94465c6dcde2abab is 50, key is test_row_0/B:col10/1733645653086/Put/seqid=0 2024-12-08T08:14:13,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645713244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645713246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645713247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645713253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645713256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741871_1047 (size=12151) 2024-12-08T08:14:13,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/45b694078bae44af94465c6dcde2abab 2024-12-08T08:14:13,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/5a666a9380c74b26a1db15d0bea1ffc9 is 50, key is test_row_0/C:col10/1733645653086/Put/seqid=0 2024-12-08T08:14:13,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741872_1048 (size=12151) 2024-12-08T08:14:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T08:14:13,352 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:13,353 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-08T08:14:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T08:14:13,357 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:13,364 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:13,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:13,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645713362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645713362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645713362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645713371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,377 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/85d8eab801ee4eee861807b922a3e863 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/85d8eab801ee4eee861807b922a3e863 2024-12-08T08:14:13,387 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/B of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 85d8eab801ee4eee861807b922a3e863(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:13,387 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:13,388 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/B, priority=13, startTime=1733645652878; duration=0sec 2024-12-08T08:14:13,389 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:13,390 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:B 2024-12-08T08:14:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T08:14:13,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T08:14:13,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:13,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:13,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:13,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:13,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:13,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645713571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645713571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645713571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645713581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T08:14:13,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T08:14:13,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:13,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:13,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:13,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:13,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:13,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/5a666a9380c74b26a1db15d0bea1ffc9 2024-12-08T08:14:13,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/e213e29eaee14af095dcb85da3efedcf as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e213e29eaee14af095dcb85da3efedcf 2024-12-08T08:14:13,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e213e29eaee14af095dcb85da3efedcf, entries=150, sequenceid=154, filesize=11.9 K 2024-12-08T08:14:13,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/45b694078bae44af94465c6dcde2abab as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/45b694078bae44af94465c6dcde2abab 2024-12-08T08:14:13,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/45b694078bae44af94465c6dcde2abab, entries=150, sequenceid=154, filesize=11.9 K 2024-12-08T08:14:13,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/5a666a9380c74b26a1db15d0bea1ffc9 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/5a666a9380c74b26a1db15d0bea1ffc9 2024-12-08T08:14:13,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/5a666a9380c74b26a1db15d0bea1ffc9, entries=150, sequenceid=154, filesize=11.9 K 2024-12-08T08:14:13,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 556ms, sequenceid=154, compaction requested=true 2024-12-08T08:14:13,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:13,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:13,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:13,761 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:13,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:13,761 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:13,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:13,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:13,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:13,762 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:13,762 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:13,762 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/A is initiating minor compaction (all files) 2024-12-08T08:14:13,762 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/B is initiating minor compaction (all files) 2024-12-08T08:14:13,762 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/A in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:13,763 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/B in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:13,763 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/85d8eab801ee4eee861807b922a3e863, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/34d919a7ab974c68b746e34817306d52, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/45b694078bae44af94465c6dcde2abab] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=35.8 K 2024-12-08T08:14:13,763 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/34a9a4ce644b460387c4328c9314782e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e4b611d78ebf4506a84dc0791f5eac4c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e213e29eaee14af095dcb85da3efedcf] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=35.8 K 2024-12-08T08:14:13,764 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34a9a4ce644b460387c4328c9314782e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733645652050 2024-12-08T08:14:13,764 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 85d8eab801ee4eee861807b922a3e863, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733645652050 2024-12-08T08:14:13,765 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4b611d78ebf4506a84dc0791f5eac4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733645652720 2024-12-08T08:14:13,765 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e213e29eaee14af095dcb85da3efedcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733645653086 2024-12-08T08:14:13,765 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 34d919a7ab974c68b746e34817306d52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733645652720 2024-12-08T08:14:13,766 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b694078bae44af94465c6dcde2abab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733645653086 2024-12-08T08:14:13,782 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:13,783 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/d4418f4f64c6438fb184de2a7c9edb6d is 50, key is test_row_0/B:col10/1733645653086/Put/seqid=0 2024-12-08T08:14:13,788 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:13,789 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/d2d2533d6b9b4aa4874e9b8cebd8060c is 50, key is test_row_0/A:col10/1733645653086/Put/seqid=0 2024-12-08T08:14:13,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741873_1049 (size=12561) 2024-12-08T08:14:13,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741874_1050 (size=12561) 2024-12-08T08:14:13,818 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/d4418f4f64c6438fb184de2a7c9edb6d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/d4418f4f64c6438fb184de2a7c9edb6d 2024-12-08T08:14:13,824 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/d2d2533d6b9b4aa4874e9b8cebd8060c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d2d2533d6b9b4aa4874e9b8cebd8060c 2024-12-08T08:14:13,833 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/B of 6e9f0e742ef7ddcc8f58c1d611d5e17b into d4418f4f64c6438fb184de2a7c9edb6d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:13,833 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:13,833 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/B, priority=13, startTime=1733645653761; duration=0sec 2024-12-08T08:14:13,833 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:13,833 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:B 2024-12-08T08:14:13,833 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:13,835 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/A of 6e9f0e742ef7ddcc8f58c1d611d5e17b into d2d2533d6b9b4aa4874e9b8cebd8060c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:13,836 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:13,836 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/A, priority=13, startTime=1733645653761; duration=0sec 2024-12-08T08:14:13,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T08:14:13,836 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:13,836 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:A 2024-12-08T08:14:13,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:13,836 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T08:14:13,836 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:13,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:13,837 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/C is initiating minor compaction (all files) 2024-12-08T08:14:13,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:13,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:13,837 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/C in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:13,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:13,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:13,837 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/1f17ec22b68c4748b43ac9b28041e37c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/de5068abaead4de0bd7050f7a2d2a037, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/5a666a9380c74b26a1db15d0bea1ffc9] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=35.8 K 2024-12-08T08:14:13,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:13,840 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f17ec22b68c4748b43ac9b28041e37c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733645652050 2024-12-08T08:14:13,841 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting de5068abaead4de0bd7050f7a2d2a037, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733645652720 2024-12-08T08:14:13,842 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a666a9380c74b26a1db15d0bea1ffc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733645653086 2024-12-08T08:14:13,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/1fafb508a1464fcd88e0b033b6c03c21 is 50, key is test_row_0/A:col10/1733645653243/Put/seqid=0 2024-12-08T08:14:13,857 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:13,857 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/193f9b6fc70d410a984e39feaaed363a is 50, key is test_row_0/C:col10/1733645653086/Put/seqid=0 2024-12-08T08:14:13,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741875_1051 (size=12151) 2024-12-08T08:14:13,862 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/1fafb508a1464fcd88e0b033b6c03c21 2024-12-08T08:14:13,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741876_1052 (size=12561) 2024-12-08T08:14:13,883 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/193f9b6fc70d410a984e39feaaed363a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/193f9b6fc70d410a984e39feaaed363a 2024-12-08T08:14:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:13,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:13,896 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/C of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 193f9b6fc70d410a984e39feaaed363a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:13,897 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:13,897 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/C, priority=13, startTime=1733645653761; duration=0sec 2024-12-08T08:14:13,897 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:13,897 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:C 2024-12-08T08:14:13,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/7250125fc58640fe91572ce17efefe3c is 50, key is test_row_0/B:col10/1733645653243/Put/seqid=0 2024-12-08T08:14:13,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645713905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645713912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645713912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645713912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:13,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:13,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741877_1053 (size=12151) 2024-12-08T08:14:13,936 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/7250125fc58640fe91572ce17efefe3c 2024-12-08T08:14:13,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/8029b92cbf84453cbf5265cdb49e3d08 is 50, key is test_row_0/C:col10/1733645653243/Put/seqid=0 2024-12-08T08:14:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T08:14:13,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741878_1054 (size=12151) 2024-12-08T08:14:14,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645714023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645714024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645714024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645714025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645714231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645714231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645714231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645714231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645714271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,277 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 6e9f0e742ef7ddcc8f58c1d611d5e17b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,399 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/8029b92cbf84453cbf5265cdb49e3d08 2024-12-08T08:14:14,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/1fafb508a1464fcd88e0b033b6c03c21 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1fafb508a1464fcd88e0b033b6c03c21 2024-12-08T08:14:14,415 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1fafb508a1464fcd88e0b033b6c03c21, entries=150, sequenceid=175, filesize=11.9 K 2024-12-08T08:14:14,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/7250125fc58640fe91572ce17efefe3c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/7250125fc58640fe91572ce17efefe3c 2024-12-08T08:14:14,424 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/7250125fc58640fe91572ce17efefe3c, entries=150, sequenceid=175, filesize=11.9 K 2024-12-08T08:14:14,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/8029b92cbf84453cbf5265cdb49e3d08 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/8029b92cbf84453cbf5265cdb49e3d08 2024-12-08T08:14:14,432 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/8029b92cbf84453cbf5265cdb49e3d08, entries=150, sequenceid=175, filesize=11.9 K 2024-12-08T08:14:14,434 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 597ms, sequenceid=175, compaction requested=false 2024-12-08T08:14:14,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:14,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:14,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-08T08:14:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-08T08:14:14,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-08T08:14:14,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0720 sec 2024-12-08T08:14:14,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0890 sec 2024-12-08T08:14:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T08:14:14,492 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:14,492 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-08T08:14:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T08:14:14,496 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:14,497 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:14,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:14,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T08:14:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:14,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:14,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:14,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:14,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:14,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/29d3eaf1aa604441b68fcb54b8a855b5 is 50, key is test_row_0/A:col10/1733645653909/Put/seqid=0 2024-12-08T08:14:14,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645714580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645714580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645714584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741879_1055 (size=12151) 2024-12-08T08:14:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645714584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T08:14:14,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T08:14:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:14,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:14,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645714691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645714691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645714700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645714701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T08:14:14,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:14,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:14,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:14,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:14,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T08:14:14,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645714900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645714901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645714911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:14,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645714911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:14,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T08:14:14,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:14,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:14,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:14,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:14,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:14,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/29d3eaf1aa604441b68fcb54b8a855b5 2024-12-08T08:14:15,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/b40f16a6af6045c598d75a141d42894e is 50, key is test_row_0/B:col10/1733645653909/Put/seqid=0 2024-12-08T08:14:15,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741880_1056 (size=12151) 2024-12-08T08:14:15,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T08:14:15,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:15,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T08:14:15,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645715211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:15,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645715211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:15,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645715220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645715220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:15,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T08:14:15,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:15,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/b40f16a6af6045c598d75a141d42894e 2024-12-08T08:14:15,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T08:14:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/531869fa473a4acf824f30a1fa957ca2 is 50, key is test_row_0/C:col10/1733645653909/Put/seqid=0 2024-12-08T08:14:15,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741881_1057 (size=12151) 2024-12-08T08:14:15,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T08:14:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T08:14:15,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645715720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:15,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645715720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:15,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645715731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645715731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:15,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T08:14:15,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:15,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/531869fa473a4acf824f30a1fa957ca2 2024-12-08T08:14:15,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/29d3eaf1aa604441b68fcb54b8a855b5 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/29d3eaf1aa604441b68fcb54b8a855b5 2024-12-08T08:14:15,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/29d3eaf1aa604441b68fcb54b8a855b5, entries=150, sequenceid=192, filesize=11.9 K 2024-12-08T08:14:15,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/b40f16a6af6045c598d75a141d42894e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/b40f16a6af6045c598d75a141d42894e 2024-12-08T08:14:15,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/b40f16a6af6045c598d75a141d42894e, entries=150, sequenceid=192, filesize=11.9 K 2024-12-08T08:14:15,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/531869fa473a4acf824f30a1fa957ca2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/531869fa473a4acf824f30a1fa957ca2 2024-12-08T08:14:15,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T08:14:15,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:15,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:15,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/531869fa473a4acf824f30a1fa957ca2, entries=150, sequenceid=192, filesize=11.9 K 2024-12-08T08:14:15,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 1370ms, sequenceid=192, compaction requested=true 2024-12-08T08:14:15,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:15,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:15,916 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:15,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:15,916 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:15,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:15,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:15,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:15,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:15,918 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:15,918 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:15,918 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/A is initiating minor compaction (all files) 2024-12-08T08:14:15,918 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/B is initiating minor compaction (all files) 2024-12-08T08:14:15,918 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/A in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,918 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d2d2533d6b9b4aa4874e9b8cebd8060c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1fafb508a1464fcd88e0b033b6c03c21, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/29d3eaf1aa604441b68fcb54b8a855b5] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=36.0 K 2024-12-08T08:14:15,918 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/B in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,918 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/d4418f4f64c6438fb184de2a7c9edb6d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/7250125fc58640fe91572ce17efefe3c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/b40f16a6af6045c598d75a141d42894e] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=36.0 K 2024-12-08T08:14:15,919 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2d2533d6b9b4aa4874e9b8cebd8060c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733645653086 2024-12-08T08:14:15,919 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting d4418f4f64c6438fb184de2a7c9edb6d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733645653086 2024-12-08T08:14:15,919 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fafb508a1464fcd88e0b033b6c03c21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733645653228 2024-12-08T08:14:15,920 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29d3eaf1aa604441b68fcb54b8a855b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733645653892 2024-12-08T08:14:15,920 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 7250125fc58640fe91572ce17efefe3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733645653228 2024-12-08T08:14:15,921 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting b40f16a6af6045c598d75a141d42894e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733645653892 2024-12-08T08:14:15,937 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#A#compaction#45 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:15,937 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#B#compaction#46 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:15,938 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/412353aea9d944a58cf4a7d394cea06e is 50, key is test_row_0/A:col10/1733645653909/Put/seqid=0 2024-12-08T08:14:15,938 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/54288e5084a7469eaf340fa50022f04e is 50, key is test_row_0/B:col10/1733645653909/Put/seqid=0 2024-12-08T08:14:15,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741882_1058 (size=12663) 2024-12-08T08:14:15,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741883_1059 (size=12663) 2024-12-08T08:14:15,967 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/54288e5084a7469eaf340fa50022f04e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/54288e5084a7469eaf340fa50022f04e 2024-12-08T08:14:15,975 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/B of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 54288e5084a7469eaf340fa50022f04e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:15,975 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:15,975 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/B, priority=13, startTime=1733645655916; duration=0sec 2024-12-08T08:14:15,975 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:15,975 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:B 2024-12-08T08:14:15,975 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:15,976 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:15,977 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/C is initiating minor compaction (all files) 2024-12-08T08:14:15,977 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/C in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:15,977 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/193f9b6fc70d410a984e39feaaed363a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/8029b92cbf84453cbf5265cdb49e3d08, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/531869fa473a4acf824f30a1fa957ca2] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=36.0 K 2024-12-08T08:14:15,977 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 193f9b6fc70d410a984e39feaaed363a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733645653086 2024-12-08T08:14:15,978 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 8029b92cbf84453cbf5265cdb49e3d08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733645653228 2024-12-08T08:14:15,978 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 531869fa473a4acf824f30a1fa957ca2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733645653892 2024-12-08T08:14:15,993 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#C#compaction#47 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:15,994 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/3a8543a73f92435ab6789b0da7465a00 is 50, key is test_row_0/C:col10/1733645653909/Put/seqid=0 2024-12-08T08:14:16,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741884_1060 (size=12663) 2024-12-08T08:14:16,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T08:14:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:16,058 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T08:14:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:16,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/475a7a9f14f24e3a9f023c5398c8a65d is 50, key is test_row_0/A:col10/1733645654582/Put/seqid=0 2024-12-08T08:14:16,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741885_1061 (size=12151) 2024-12-08T08:14:16,080 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/475a7a9f14f24e3a9f023c5398c8a65d 2024-12-08T08:14:16,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/bcb820fba84846918ef063f974910115 is 50, key is test_row_0/B:col10/1733645654582/Put/seqid=0 2024-12-08T08:14:16,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741886_1062 (size=12151) 2024-12-08T08:14:16,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:16,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645716331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:16,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:16,367 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/412353aea9d944a58cf4a7d394cea06e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/412353aea9d944a58cf4a7d394cea06e 2024-12-08T08:14:16,375 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/A of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 412353aea9d944a58cf4a7d394cea06e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:16,375 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:16,375 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/A, priority=13, startTime=1733645655916; duration=0sec 2024-12-08T08:14:16,375 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:16,375 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:A 2024-12-08T08:14:16,410 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/3a8543a73f92435ab6789b0da7465a00 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/3a8543a73f92435ab6789b0da7465a00 2024-12-08T08:14:16,418 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/C of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 3a8543a73f92435ab6789b0da7465a00(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:16,418 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:16,418 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/C, priority=13, startTime=1733645655916; duration=0sec 2024-12-08T08:14:16,418 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:16,418 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:C 2024-12-08T08:14:16,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645716441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:16,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:16,504 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/bcb820fba84846918ef063f974910115 2024-12-08T08:14:16,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/7993062e0c894ea7acb33b7c4baca106 is 50, key is test_row_0/C:col10/1733645654582/Put/seqid=0 2024-12-08T08:14:16,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741887_1063 (size=12151) 2024-12-08T08:14:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T08:14:16,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645716651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:16,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645716740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:16,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645716741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:16,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645716741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:16,744 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_0' in 6e9f0e742ef7ddcc8f58c1d611d5e17b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:16,744 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 6e9f0e742ef7ddcc8f58c1d611d5e17b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,745 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 6e9f0e742ef7ddcc8f58c1d611d5e17b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:16,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645716750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:16,753 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 6e9f0e742ef7ddcc8f58c1d611d5e17b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:16,921 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/7993062e0c894ea7acb33b7c4baca106 2024-12-08T08:14:16,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/475a7a9f14f24e3a9f023c5398c8a65d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/475a7a9f14f24e3a9f023c5398c8a65d 2024-12-08T08:14:16,935 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/475a7a9f14f24e3a9f023c5398c8a65d, entries=150, sequenceid=210, filesize=11.9 K 2024-12-08T08:14:16,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/bcb820fba84846918ef063f974910115 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/bcb820fba84846918ef063f974910115 2024-12-08T08:14:16,942 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/bcb820fba84846918ef063f974910115, entries=150, sequenceid=210, filesize=11.9 K 2024-12-08T08:14:16,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/7993062e0c894ea7acb33b7c4baca106 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/7993062e0c894ea7acb33b7c4baca106 2024-12-08T08:14:16,952 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/7993062e0c894ea7acb33b7c4baca106, entries=150, sequenceid=210, filesize=11.9 K 2024-12-08T08:14:16,953 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 896ms, sequenceid=210, compaction requested=false 2024-12-08T08:14:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-08T08:14:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-08T08:14:16,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-08T08:14:16,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4580 sec 2024-12-08T08:14:16,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.4650 sec 2024-12-08T08:14:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:16,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T08:14:16,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:16,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:16,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:16,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:16,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:16,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:16,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/e36d76dac77341bfa9dd73a39824d21b is 50, key is test_row_0/A:col10/1733645656329/Put/seqid=0 2024-12-08T08:14:16,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741888_1064 (size=14541) 2024-12-08T08:14:17,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645717030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645717140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:17,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645717351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:17,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:17,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:17,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:17,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/e36d76dac77341bfa9dd73a39824d21b 2024-12-08T08:14:17,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/309bf5ef0b344817b11c935307a945e2 is 50, key is test_row_0/B:col10/1733645656329/Put/seqid=0 2024-12-08T08:14:17,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741889_1065 (size=12151) 2024-12-08T08:14:17,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645717660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:17,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/309bf5ef0b344817b11c935307a945e2 2024-12-08T08:14:17,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/64200a2eb0e7476caa53694e4ebf6850 is 50, key is test_row_0/C:col10/1733645656329/Put/seqid=0 2024-12-08T08:14:17,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741890_1066 (size=12151) 2024-12-08T08:14:18,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645718170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:18,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/64200a2eb0e7476caa53694e4ebf6850 2024-12-08T08:14:18,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/e36d76dac77341bfa9dd73a39824d21b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e36d76dac77341bfa9dd73a39824d21b 2024-12-08T08:14:18,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e36d76dac77341bfa9dd73a39824d21b, entries=200, sequenceid=228, filesize=14.2 K 2024-12-08T08:14:18,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/309bf5ef0b344817b11c935307a945e2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/309bf5ef0b344817b11c935307a945e2 2024-12-08T08:14:18,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/309bf5ef0b344817b11c935307a945e2, entries=150, sequenceid=228, filesize=11.9 K 2024-12-08T08:14:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/64200a2eb0e7476caa53694e4ebf6850 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/64200a2eb0e7476caa53694e4ebf6850 2024-12-08T08:14:18,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/64200a2eb0e7476caa53694e4ebf6850, entries=150, sequenceid=228, filesize=11.9 K 2024-12-08T08:14:18,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 1274ms, sequenceid=228, compaction requested=true 2024-12-08T08:14:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:18,238 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:18,238 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:18,239 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:18,239 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:18,239 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/A is initiating minor compaction (all files) 2024-12-08T08:14:18,239 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/B is initiating minor compaction (all files) 2024-12-08T08:14:18,239 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/A in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:18,239 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/B in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:18,239 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/54288e5084a7469eaf340fa50022f04e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/bcb820fba84846918ef063f974910115, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/309bf5ef0b344817b11c935307a945e2] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=36.1 K 2024-12-08T08:14:18,239 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/412353aea9d944a58cf4a7d394cea06e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/475a7a9f14f24e3a9f023c5398c8a65d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e36d76dac77341bfa9dd73a39824d21b] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=38.4 K 2024-12-08T08:14:18,240 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 54288e5084a7469eaf340fa50022f04e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733645653892 2024-12-08T08:14:18,240 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 412353aea9d944a58cf4a7d394cea06e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733645653892 2024-12-08T08:14:18,240 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting bcb820fba84846918ef063f974910115, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733645654574 2024-12-08T08:14:18,240 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 475a7a9f14f24e3a9f023c5398c8a65d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733645654574 2024-12-08T08:14:18,241 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e36d76dac77341bfa9dd73a39824d21b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733645656316 2024-12-08T08:14:18,241 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 309bf5ef0b344817b11c935307a945e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733645656316 2024-12-08T08:14:18,249 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#B#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:18,250 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/807a7c0f5b654ddda8517d28686df137 is 50, key is test_row_0/B:col10/1733645656329/Put/seqid=0 2024-12-08T08:14:18,252 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#A#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:18,253 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/b0cb1a5da44842edb45f6e0d3a55ae36 is 50, key is test_row_0/A:col10/1733645656329/Put/seqid=0 2024-12-08T08:14:18,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741892_1068 (size=12765) 2024-12-08T08:14:18,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741891_1067 (size=12765) 2024-12-08T08:14:18,274 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/b0cb1a5da44842edb45f6e0d3a55ae36 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/b0cb1a5da44842edb45f6e0d3a55ae36 2024-12-08T08:14:18,276 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/807a7c0f5b654ddda8517d28686df137 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/807a7c0f5b654ddda8517d28686df137 2024-12-08T08:14:18,282 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/A of 6e9f0e742ef7ddcc8f58c1d611d5e17b into b0cb1a5da44842edb45f6e0d3a55ae36(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:18,282 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:18,283 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/A, priority=13, startTime=1733645658238; duration=0sec 2024-12-08T08:14:18,283 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:18,283 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:A 2024-12-08T08:14:18,283 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:18,283 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/B of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 807a7c0f5b654ddda8517d28686df137(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:18,283 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:18,283 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/B, priority=13, startTime=1733645658238; duration=0sec 2024-12-08T08:14:18,284 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:18,284 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:B 2024-12-08T08:14:18,285 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:18,285 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/C is initiating minor compaction (all files) 2024-12-08T08:14:18,285 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/C in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:18,286 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/3a8543a73f92435ab6789b0da7465a00, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/7993062e0c894ea7acb33b7c4baca106, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/64200a2eb0e7476caa53694e4ebf6850] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=36.1 K 2024-12-08T08:14:18,286 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a8543a73f92435ab6789b0da7465a00, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733645653892 2024-12-08T08:14:18,287 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7993062e0c894ea7acb33b7c4baca106, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733645654574 2024-12-08T08:14:18,287 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64200a2eb0e7476caa53694e4ebf6850, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733645656316 2024-12-08T08:14:18,298 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#C#compaction#56 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:18,299 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/d479e95524e548fb8b949ae3cb702d47 is 50, key is test_row_0/C:col10/1733645656329/Put/seqid=0 2024-12-08T08:14:18,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741893_1069 (size=12765) 2024-12-08T08:14:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T08:14:18,651 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:18,652 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-08T08:14:18,655 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T08:14:18,656 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:18,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:18,714 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/d479e95524e548fb8b949ae3cb702d47 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/d479e95524e548fb8b949ae3cb702d47 2024-12-08T08:14:18,722 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/C of 6e9f0e742ef7ddcc8f58c1d611d5e17b into d479e95524e548fb8b949ae3cb702d47(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:18,722 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:18,722 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/C, priority=13, startTime=1733645658238; duration=0sec 2024-12-08T08:14:18,722 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:18,722 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:C 2024-12-08T08:14:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T08:14:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:18,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T08:14:18,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:18,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:18,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:18,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:18,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:18,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:18,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/cf776c830cff4befaf04af575f906779 is 50, key is test_row_0/A:col10/1733645657014/Put/seqid=0 2024-12-08T08:14:18,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741894_1070 (size=16931) 2024-12-08T08:14:18,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/cf776c830cff4befaf04af575f906779 2024-12-08T08:14:18,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/fb1b7df0dde4433c94d65d97e4bf3b98 is 50, key is test_row_0/B:col10/1733645657014/Put/seqid=0 2024-12-08T08:14:18,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741895_1071 (size=12151) 2024-12-08T08:14:18,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645718786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:18,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645718787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:18,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645718787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:18,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645718788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:18,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T08:14:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:18,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:18,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:18,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645718901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:18,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:18,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645718901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645718901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:18,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645718901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:18,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T08:14:18,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:18,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:18,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:18,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:18,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T08:14:19,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645719110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:19,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645719110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:19,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645719111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645719111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:19,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T08:14:19,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:19,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:19,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:19,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:19,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645719191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:19,192 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_0' in 6e9f0e742ef7ddcc8f58c1d611d5e17b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:19,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/fb1b7df0dde4433c94d65d97e4bf3b98 2024-12-08T08:14:19,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/16028823196746e8904b9c794be981b1 is 50, key is test_row_0/C:col10/1733645657014/Put/seqid=0 2024-12-08T08:14:19,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741896_1072 (size=12151) 2024-12-08T08:14:19,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T08:14:19,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:19,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:19,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:19,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:19,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T08:14:19,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645719421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:19,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:19,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645719421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645719421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:19,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645719421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:19,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T08:14:19,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:19,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:19,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:19,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:19,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:19,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T08:14:19,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:19,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:19,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:19,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:19,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:19,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/16028823196746e8904b9c794be981b1 2024-12-08T08:14:19,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/cf776c830cff4befaf04af575f906779 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/cf776c830cff4befaf04af575f906779 2024-12-08T08:14:19,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/cf776c830cff4befaf04af575f906779, entries=250, sequenceid=249, filesize=16.5 K 2024-12-08T08:14:19,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/fb1b7df0dde4433c94d65d97e4bf3b98 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/fb1b7df0dde4433c94d65d97e4bf3b98 2024-12-08T08:14:19,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/fb1b7df0dde4433c94d65d97e4bf3b98, entries=150, sequenceid=249, filesize=11.9 K 2024-12-08T08:14:19,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/16028823196746e8904b9c794be981b1 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/16028823196746e8904b9c794be981b1 2024-12-08T08:14:19,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/16028823196746e8904b9c794be981b1, entries=150, sequenceid=249, filesize=11.9 K 2024-12-08T08:14:19,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 881ms, sequenceid=249, compaction requested=false 2024-12-08T08:14:19,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:19,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T08:14:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:19,732 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T08:14:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:19,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:19,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/6ad2a86b228a4fd7adfc96602a73712e is 50, key is test_row_0/A:col10/1733645658773/Put/seqid=0 2024-12-08T08:14:19,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741897_1073 (size=12251) 2024-12-08T08:14:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T08:14:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:19,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:19,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645719952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:19,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645719952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:19,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645719954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:19,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:19,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645719956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:20,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645720061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:20,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645720061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645720061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:20,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645720070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:20,148 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/6ad2a86b228a4fd7adfc96602a73712e 2024-12-08T08:14:20,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/9c011e8788c7472c950bb1c28bbedf01 is 50, key is test_row_0/B:col10/1733645658773/Put/seqid=0 2024-12-08T08:14:20,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741898_1074 (size=12251) 2024-12-08T08:14:20,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645720270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:20,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645720270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645720271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:20,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645720281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:20,565 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/9c011e8788c7472c950bb1c28bbedf01 2024-12-08T08:14:20,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/6881ca2e3ad04648bc671403dcac40f0 is 50, key is test_row_0/C:col10/1733645658773/Put/seqid=0 2024-12-08T08:14:20,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741899_1075 (size=12251) 2024-12-08T08:14:20,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645720581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:20,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645720581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645720581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:20,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645720590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T08:14:20,985 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/6881ca2e3ad04648bc671403dcac40f0 2024-12-08T08:14:20,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/6ad2a86b228a4fd7adfc96602a73712e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/6ad2a86b228a4fd7adfc96602a73712e 2024-12-08T08:14:20,996 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/6ad2a86b228a4fd7adfc96602a73712e, entries=150, sequenceid=263, filesize=12.0 K 2024-12-08T08:14:20,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/9c011e8788c7472c950bb1c28bbedf01 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/9c011e8788c7472c950bb1c28bbedf01 2024-12-08T08:14:21,006 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/9c011e8788c7472c950bb1c28bbedf01, entries=150, sequenceid=263, filesize=12.0 K 2024-12-08T08:14:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/6881ca2e3ad04648bc671403dcac40f0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/6881ca2e3ad04648bc671403dcac40f0 2024-12-08T08:14:21,012 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/6881ca2e3ad04648bc671403dcac40f0, entries=150, sequenceid=263, filesize=12.0 K 2024-12-08T08:14:21,013 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 1281ms, sequenceid=263, compaction requested=true 2024-12-08T08:14:21,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:21,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:21,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-08T08:14:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-08T08:14:21,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-08T08:14:21,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3590 sec 2024-12-08T08:14:21,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.3660 sec 2024-12-08T08:14:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:21,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T08:14:21,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:21,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:21,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:21,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:21,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:21,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:21,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645721109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645721110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645721111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645721114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/9a34d6daa37c4490a3b1dbd8d73720a1 is 50, key is test_row_0/A:col10/1733645661092/Put/seqid=0 2024-12-08T08:14:21,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741900_1076 (size=12301) 2024-12-08T08:14:21,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/9a34d6daa37c4490a3b1dbd8d73720a1 2024-12-08T08:14:21,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/63890a71f9c04be1b146ba7cf130d3fc is 50, key is test_row_0/B:col10/1733645661092/Put/seqid=0 2024-12-08T08:14:21,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741901_1077 (size=12301) 2024-12-08T08:14:21,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/63890a71f9c04be1b146ba7cf130d3fc 2024-12-08T08:14:21,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/bb5881323453472b868474515b73c9f9 is 50, key is test_row_0/C:col10/1733645661092/Put/seqid=0 2024-12-08T08:14:21,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741902_1078 (size=12301) 2024-12-08T08:14:21,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/bb5881323453472b868474515b73c9f9 2024-12-08T08:14:21,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/9a34d6daa37c4490a3b1dbd8d73720a1 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/9a34d6daa37c4490a3b1dbd8d73720a1 2024-12-08T08:14:21,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/9a34d6daa37c4490a3b1dbd8d73720a1, entries=150, sequenceid=283, filesize=12.0 K 2024-12-08T08:14:21,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/63890a71f9c04be1b146ba7cf130d3fc as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/63890a71f9c04be1b146ba7cf130d3fc 2024-12-08T08:14:21,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645721210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,212 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_0' in 6e9f0e742ef7ddcc8f58c1d611d5e17b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/63890a71f9c04be1b146ba7cf130d3fc, entries=150, sequenceid=283, filesize=12.0 K 2024-12-08T08:14:21,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/bb5881323453472b868474515b73c9f9 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/bb5881323453472b868474515b73c9f9 2024-12-08T08:14:21,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/bb5881323453472b868474515b73c9f9, entries=150, sequenceid=283, filesize=12.0 K 2024-12-08T08:14:21,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 130ms, sequenceid=283, compaction requested=true 2024-12-08T08:14:21,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:21,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:21,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:21,223 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:21,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:21,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:21,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:21,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:21,223 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:21,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T08:14:21,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:21,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:21,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:21,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:21,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:21,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:21,227 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49468 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:21,227 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/B is initiating minor compaction (all files) 2024-12-08T08:14:21,227 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54248 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:21,227 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/B in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:21,227 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/A is initiating minor compaction (all files) 2024-12-08T08:14:21,227 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/A in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:21,227 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/807a7c0f5b654ddda8517d28686df137, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/fb1b7df0dde4433c94d65d97e4bf3b98, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/9c011e8788c7472c950bb1c28bbedf01, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/63890a71f9c04be1b146ba7cf130d3fc] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=48.3 K 2024-12-08T08:14:21,227 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/b0cb1a5da44842edb45f6e0d3a55ae36, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/cf776c830cff4befaf04af575f906779, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/6ad2a86b228a4fd7adfc96602a73712e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/9a34d6daa37c4490a3b1dbd8d73720a1] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=53.0 K 2024-12-08T08:14:21,228 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0cb1a5da44842edb45f6e0d3a55ae36, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733645656316 2024-12-08T08:14:21,228 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 807a7c0f5b654ddda8517d28686df137, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733645656316 2024-12-08T08:14:21,228 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf776c830cff4befaf04af575f906779, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733645657014 2024-12-08T08:14:21,229 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting fb1b7df0dde4433c94d65d97e4bf3b98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733645657014 2024-12-08T08:14:21,229 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ad2a86b228a4fd7adfc96602a73712e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733645658773 2024-12-08T08:14:21,229 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a34d6daa37c4490a3b1dbd8d73720a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733645659952 2024-12-08T08:14:21,230 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c011e8788c7472c950bb1c28bbedf01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733645658773 2024-12-08T08:14:21,231 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 63890a71f9c04be1b146ba7cf130d3fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733645659952 2024-12-08T08:14:21,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/4730ca4eed3f4d06a2d0bc1be3530c2a is 50, key is test_row_0/A:col10/1733645661104/Put/seqid=0 2024-12-08T08:14:21,258 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:21,259 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/3af11c8a0608423095c3b561ee421d6b is 50, key is test_row_0/A:col10/1733645661092/Put/seqid=0 2024-12-08T08:14:21,259 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#B#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:21,260 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/606a178f4b17411499d215925662c1ce is 50, key is test_row_0/B:col10/1733645661092/Put/seqid=0 2024-12-08T08:14:21,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741903_1079 (size=12301) 2024-12-08T08:14:21,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645721256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645721254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645721264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645721264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741904_1080 (size=13051) 2024-12-08T08:14:21,283 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/3af11c8a0608423095c3b561ee421d6b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/3af11c8a0608423095c3b561ee421d6b 2024-12-08T08:14:21,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741905_1081 (size=13051) 2024-12-08T08:14:21,350 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/A of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 3af11c8a0608423095c3b561ee421d6b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:21,350 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:21,350 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/A, priority=12, startTime=1733645661223; duration=0sec 2024-12-08T08:14:21,350 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:21,350 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:A 2024-12-08T08:14:21,350 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:21,352 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49468 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:21,352 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/606a178f4b17411499d215925662c1ce as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/606a178f4b17411499d215925662c1ce 2024-12-08T08:14:21,352 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/C is initiating minor compaction (all files) 2024-12-08T08:14:21,352 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/C in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:21,353 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/d479e95524e548fb8b949ae3cb702d47, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/16028823196746e8904b9c794be981b1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/6881ca2e3ad04648bc671403dcac40f0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/bb5881323453472b868474515b73c9f9] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=48.3 K 2024-12-08T08:14:21,353 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting d479e95524e548fb8b949ae3cb702d47, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733645656316 2024-12-08T08:14:21,353 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16028823196746e8904b9c794be981b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733645657014 2024-12-08T08:14:21,354 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6881ca2e3ad04648bc671403dcac40f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733645658773 2024-12-08T08:14:21,354 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb5881323453472b868474515b73c9f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733645659952 2024-12-08T08:14:21,362 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/B of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 606a178f4b17411499d215925662c1ce(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:21,363 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:21,363 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/B, priority=12, startTime=1733645661223; duration=0sec 2024-12-08T08:14:21,363 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:21,363 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:B 2024-12-08T08:14:21,367 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:21,368 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/f1eb01f3ef5a433793dca103a19a70f3 is 50, key is test_row_0/C:col10/1733645661092/Put/seqid=0 2024-12-08T08:14:21,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645721371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645721371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741906_1082 (size=13051) 2024-12-08T08:14:21,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645721380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645721380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645721581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645721581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645721592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645721592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/4730ca4eed3f4d06a2d0bc1be3530c2a 2024-12-08T08:14:21,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/3a545c208797480a8cceb403ebe99d08 is 50, key is test_row_0/B:col10/1733645661104/Put/seqid=0 2024-12-08T08:14:21,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741907_1083 (size=12301) 2024-12-08T08:14:21,781 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/f1eb01f3ef5a433793dca103a19a70f3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/f1eb01f3ef5a433793dca103a19a70f3 2024-12-08T08:14:21,797 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/C of 6e9f0e742ef7ddcc8f58c1d611d5e17b into f1eb01f3ef5a433793dca103a19a70f3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:21,797 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:21,797 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/C, priority=12, startTime=1733645661223; duration=0sec 2024-12-08T08:14:21,797 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:21,797 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:C 2024-12-08T08:14:21,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645721891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645721891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645721910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645721910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:22,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/3a545c208797480a8cceb403ebe99d08 2024-12-08T08:14:22,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/cbf63a59d4a74a69acdf428ddb726a76 is 50, key is test_row_0/C:col10/1733645661104/Put/seqid=0 2024-12-08T08:14:22,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741908_1084 (size=12301) 2024-12-08T08:14:22,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645722401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:22,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645722401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:22,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645722420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:22,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645722421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:22,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/cbf63a59d4a74a69acdf428ddb726a76 2024-12-08T08:14:22,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/4730ca4eed3f4d06a2d0bc1be3530c2a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/4730ca4eed3f4d06a2d0bc1be3530c2a 2024-12-08T08:14:22,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/4730ca4eed3f4d06a2d0bc1be3530c2a, entries=150, sequenceid=296, filesize=12.0 K 2024-12-08T08:14:22,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/3a545c208797480a8cceb403ebe99d08 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/3a545c208797480a8cceb403ebe99d08 2024-12-08T08:14:22,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/3a545c208797480a8cceb403ebe99d08, entries=150, sequenceid=296, filesize=12.0 K 2024-12-08T08:14:22,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/cbf63a59d4a74a69acdf428ddb726a76 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/cbf63a59d4a74a69acdf428ddb726a76 2024-12-08T08:14:22,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/cbf63a59d4a74a69acdf428ddb726a76, entries=150, sequenceid=296, filesize=12.0 K 2024-12-08T08:14:22,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 1373ms, sequenceid=296, compaction requested=false 2024-12-08T08:14:22,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T08:14:22,821 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:22,822 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-08T08:14:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T08:14:22,825 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:22,826 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:22,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T08:14:22,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-08T08:14:22,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:22,980 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T08:14:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:22,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:22,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/be08f410a1ae4a4ab81057de80a888e6 is 50, key is test_row_0/A:col10/1733645661252/Put/seqid=0 2024-12-08T08:14:23,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741909_1085 (size=12301) 2024-12-08T08:14:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T08:14:23,407 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/be08f410a1ae4a4ab81057de80a888e6 2024-12-08T08:14:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:23,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:23,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/c6dc46728b21428d968d9a3b6309224c is 50, key is test_row_0/B:col10/1733645661252/Put/seqid=0 2024-12-08T08:14:23,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645723431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:23,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645723432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:23,437 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 6e9f0e742ef7ddcc8f58c1d611d5e17b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:23,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645723434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645723436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T08:14:23,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741910_1086 (size=12301) 2024-12-08T08:14:23,483 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/c6dc46728b21428d968d9a3b6309224c 2024-12-08T08:14:23,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/1a9f1a71688748eda3cb7b2157007cca is 50, key is test_row_0/C:col10/1733645661252/Put/seqid=0 2024-12-08T08:14:23,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741911_1087 (size=12301) 2024-12-08T08:14:23,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645723540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:23,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645723540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:23,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645723551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:23,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645723750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645723750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:23,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645723761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:23,923 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/1a9f1a71688748eda3cb7b2157007cca 2024-12-08T08:14:23,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/be08f410a1ae4a4ab81057de80a888e6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/be08f410a1ae4a4ab81057de80a888e6 2024-12-08T08:14:23,937 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/be08f410a1ae4a4ab81057de80a888e6, entries=150, sequenceid=318, filesize=12.0 K 2024-12-08T08:14:23,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/c6dc46728b21428d968d9a3b6309224c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/c6dc46728b21428d968d9a3b6309224c 2024-12-08T08:14:23,943 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/c6dc46728b21428d968d9a3b6309224c, entries=150, sequenceid=318, filesize=12.0 K 2024-12-08T08:14:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/1a9f1a71688748eda3cb7b2157007cca as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/1a9f1a71688748eda3cb7b2157007cca 2024-12-08T08:14:23,953 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/1a9f1a71688748eda3cb7b2157007cca, entries=150, sequenceid=318, filesize=12.0 K 2024-12-08T08:14:23,954 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 975ms, sequenceid=318, compaction requested=true 2024-12-08T08:14:23,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:23,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:23,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-08T08:14:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-08T08:14:23,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-08T08:14:23,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1300 sec 2024-12-08T08:14:23,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.1370 sec 2024-12-08T08:14:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T08:14:23,961 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:23,961 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-08T08:14:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T08:14:23,963 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:23,964 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:23,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:24,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T08:14:24,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:24,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:24,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:24,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:24,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:24,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:24,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/4897509d03d941e19406a08beb6548b0 is 50, key is test_row_0/A:col10/1733645664061/Put/seqid=0 2024-12-08T08:14:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T08:14:24,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741912_1088 (size=17181) 2024-12-08T08:14:24,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645724090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645724092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645724093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T08:14:24,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:24,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:24,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645724201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645724207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645724207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T08:14:24,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:24,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:24,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:24,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:24,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T08:14:24,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645724410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T08:14:24,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645724420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:24,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645724420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:24,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/4897509d03d941e19406a08beb6548b0 2024-12-08T08:14:24,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/7fadc42c065a41f5b749ecce2f8b4eba is 50, key is test_row_0/B:col10/1733645664061/Put/seqid=0 2024-12-08T08:14:24,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741913_1089 (size=12301) 2024-12-08T08:14:24,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/7fadc42c065a41f5b749ecce2f8b4eba 2024-12-08T08:14:24,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/186fcbf4f64647b5bbbbbdb833b80160 is 50, key is test_row_0/C:col10/1733645664061/Put/seqid=0 2024-12-08T08:14:24,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741914_1090 (size=12301) 2024-12-08T08:14:24,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/186fcbf4f64647b5bbbbbdb833b80160 2024-12-08T08:14:24,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/4897509d03d941e19406a08beb6548b0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/4897509d03d941e19406a08beb6548b0 2024-12-08T08:14:24,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/4897509d03d941e19406a08beb6548b0, entries=250, sequenceid=333, filesize=16.8 K 2024-12-08T08:14:24,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/7fadc42c065a41f5b749ecce2f8b4eba as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/7fadc42c065a41f5b749ecce2f8b4eba 2024-12-08T08:14:24,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/7fadc42c065a41f5b749ecce2f8b4eba, entries=150, sequenceid=333, filesize=12.0 K 2024-12-08T08:14:24,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/186fcbf4f64647b5bbbbbdb833b80160 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/186fcbf4f64647b5bbbbbdb833b80160 2024-12-08T08:14:24,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/186fcbf4f64647b5bbbbbdb833b80160, entries=150, sequenceid=333, filesize=12.0 K 2024-12-08T08:14:24,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 468ms, sequenceid=333, compaction requested=true 2024-12-08T08:14:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:24,531 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:24,531 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:24,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:24,535 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:24,535 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/A is initiating minor compaction (all files) 2024-12-08T08:14:24,535 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/A in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:24,535 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/3af11c8a0608423095c3b561ee421d6b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/4730ca4eed3f4d06a2d0bc1be3530c2a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/be08f410a1ae4a4ab81057de80a888e6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/4897509d03d941e19406a08beb6548b0] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=53.5 K 2024-12-08T08:14:24,536 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:24,536 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/B is initiating minor compaction (all files) 2024-12-08T08:14:24,536 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3af11c8a0608423095c3b561ee421d6b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733645659952 2024-12-08T08:14:24,536 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/B in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:24,536 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/606a178f4b17411499d215925662c1ce, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/3a545c208797480a8cceb403ebe99d08, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/c6dc46728b21428d968d9a3b6309224c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/7fadc42c065a41f5b749ecce2f8b4eba] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=48.8 K 2024-12-08T08:14:24,536 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 606a178f4b17411499d215925662c1ce, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733645659952 2024-12-08T08:14:24,536 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4730ca4eed3f4d06a2d0bc1be3530c2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733645661102 2024-12-08T08:14:24,537 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a545c208797480a8cceb403ebe99d08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733645661102 2024-12-08T08:14:24,537 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting be08f410a1ae4a4ab81057de80a888e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733645661252 2024-12-08T08:14:24,537 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4897509d03d941e19406a08beb6548b0, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733645663429 2024-12-08T08:14:24,537 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting c6dc46728b21428d968d9a3b6309224c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733645661252 2024-12-08T08:14:24,537 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fadc42c065a41f5b749ecce2f8b4eba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733645663429 2024-12-08T08:14:24,551 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:24,552 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/76b6c802d7d843e1ad69e76ae54a041b is 50, key is test_row_0/B:col10/1733645664061/Put/seqid=0 2024-12-08T08:14:24,559 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#A#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:24,560 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/29bbcc7ab34045a89434e184ffd847cd is 50, key is test_row_0/A:col10/1733645664061/Put/seqid=0 2024-12-08T08:14:24,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741916_1092 (size=13187) 2024-12-08T08:14:24,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741915_1091 (size=13187) 2024-12-08T08:14:24,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T08:14:24,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:24,578 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T08:14:24,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:24,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:24,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:24,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:24,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:24,579 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/29bbcc7ab34045a89434e184ffd847cd as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/29bbcc7ab34045a89434e184ffd847cd 2024-12-08T08:14:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:24,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/8c46649d2094471c844e22136e15cc75 is 50, key is test_row_0/A:col10/1733645664089/Put/seqid=0 2024-12-08T08:14:24,590 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/76b6c802d7d843e1ad69e76ae54a041b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/76b6c802d7d843e1ad69e76ae54a041b 2024-12-08T08:14:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T08:14:24,595 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/A of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 29bbcc7ab34045a89434e184ffd847cd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:24,596 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:24,596 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/A, priority=12, startTime=1733645664531; duration=0sec 2024-12-08T08:14:24,596 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:24,596 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:A 2024-12-08T08:14:24,596 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:24,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741917_1093 (size=12301) 2024-12-08T08:14:24,598 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/8c46649d2094471c844e22136e15cc75 2024-12-08T08:14:24,600 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:24,600 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/C is initiating minor compaction (all files) 2024-12-08T08:14:24,600 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/C in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:24,600 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/f1eb01f3ef5a433793dca103a19a70f3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/cbf63a59d4a74a69acdf428ddb726a76, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/1a9f1a71688748eda3cb7b2157007cca, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/186fcbf4f64647b5bbbbbdb833b80160] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=48.8 K 2024-12-08T08:14:24,602 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/B of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 76b6c802d7d843e1ad69e76ae54a041b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:24,602 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1eb01f3ef5a433793dca103a19a70f3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733645659952 2024-12-08T08:14:24,602 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:24,602 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/B, priority=12, startTime=1733645664531; duration=0sec 2024-12-08T08:14:24,602 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:24,602 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:B 2024-12-08T08:14:24,602 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbf63a59d4a74a69acdf428ddb726a76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733645661102 2024-12-08T08:14:24,603 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a9f1a71688748eda3cb7b2157007cca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733645661252 2024-12-08T08:14:24,604 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 186fcbf4f64647b5bbbbbdb833b80160, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733645663429 2024-12-08T08:14:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/2b4d497153d04006a3c6981d8e4df9d3 is 50, key is test_row_0/B:col10/1733645664089/Put/seqid=0 2024-12-08T08:14:24,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741918_1094 (size=12301) 2024-12-08T08:14:24,624 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#C#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:24,625 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/b921b2d3a26b4ee082bc603426c94479 is 50, key is test_row_0/C:col10/1733645664061/Put/seqid=0 2024-12-08T08:14:24,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741919_1095 (size=13187) 2024-12-08T08:14:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:24,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:24,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645724740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645724743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645724744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645724851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645724851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645724851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,025 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/2b4d497153d04006a3c6981d8e4df9d3 2024-12-08T08:14:25,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/91404337f69843348a7094c19262dc1c is 50, key is test_row_0/C:col10/1733645664089/Put/seqid=0 2024-12-08T08:14:25,038 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/b921b2d3a26b4ee082bc603426c94479 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/b921b2d3a26b4ee082bc603426c94479 2024-12-08T08:14:25,046 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/C of 6e9f0e742ef7ddcc8f58c1d611d5e17b into b921b2d3a26b4ee082bc603426c94479(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:25,046 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:25,046 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/C, priority=12, startTime=1733645664531; duration=0sec 2024-12-08T08:14:25,046 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:25,046 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:C 2024-12-08T08:14:25,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741920_1096 (size=12301) 2024-12-08T08:14:25,050 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/91404337f69843348a7094c19262dc1c 2024-12-08T08:14:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/8c46649d2094471c844e22136e15cc75 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/8c46649d2094471c844e22136e15cc75 2024-12-08T08:14:25,063 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/8c46649d2094471c844e22136e15cc75, entries=150, sequenceid=350, filesize=12.0 K 2024-12-08T08:14:25,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645725061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645725061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645725061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/2b4d497153d04006a3c6981d8e4df9d3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/2b4d497153d04006a3c6981d8e4df9d3 2024-12-08T08:14:25,073 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/2b4d497153d04006a3c6981d8e4df9d3, entries=150, sequenceid=350, filesize=12.0 K 2024-12-08T08:14:25,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/91404337f69843348a7094c19262dc1c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/91404337f69843348a7094c19262dc1c 2024-12-08T08:14:25,085 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/91404337f69843348a7094c19262dc1c, entries=150, sequenceid=350, filesize=12.0 K 2024-12-08T08:14:25,086 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 508ms, sequenceid=350, compaction requested=false 2024-12-08T08:14:25,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:25,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:25,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-08T08:14:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-08T08:14:25,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-08T08:14:25,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1240 sec 2024-12-08T08:14:25,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.1300 sec 2024-12-08T08:14:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T08:14:25,101 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:25,101 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-08T08:14:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T08:14:25,103 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:25,104 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:25,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T08:14:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:25,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T08:14:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:25,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/0bbb817bfe194166a5ddfe42d0534dd8 is 50, key is test_row_0/A:col10/1733645665241/Put/seqid=0 2024-12-08T08:14:25,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741921_1097 (size=14741) 2024-12-08T08:14:25,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T08:14:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:25,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/0bbb817bfe194166a5ddfe42d0534dd8 2024-12-08T08:14:25,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/8beb8e7818804300959cf4959624b7f1 is 50, key is test_row_0/B:col10/1733645665241/Put/seqid=0 2024-12-08T08:14:25,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741922_1098 (size=12301) 2024-12-08T08:14:25,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645725293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645725371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645725371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645725371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645725401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T08:14:25,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:25,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:25,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:25,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T08:14:25,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40760 deadline: 1733645725450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,452 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 6e9f0e742ef7ddcc8f58c1d611d5e17b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T08:14:25,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:25,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:25,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:25,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645725611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/8beb8e7818804300959cf4959624b7f1 2024-12-08T08:14:25,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/85045a8ec5fd481abb4277383c52fbe2 is 50, key is test_row_0/C:col10/1733645665241/Put/seqid=0 2024-12-08T08:14:25,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741923_1099 (size=12301) 2024-12-08T08:14:25,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T08:14:25,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:25,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:25,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:25,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T08:14:25,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T08:14:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:25,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:25,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645725881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645725881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645725881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:25,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645725920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:26,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T08:14:26,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:26,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:26,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/85045a8ec5fd481abb4277383c52fbe2 2024-12-08T08:14:26,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/0bbb817bfe194166a5ddfe42d0534dd8 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/0bbb817bfe194166a5ddfe42d0534dd8 2024-12-08T08:14:26,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/0bbb817bfe194166a5ddfe42d0534dd8, entries=200, sequenceid=369, filesize=14.4 K 2024-12-08T08:14:26,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/8beb8e7818804300959cf4959624b7f1 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/8beb8e7818804300959cf4959624b7f1 2024-12-08T08:14:26,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/8beb8e7818804300959cf4959624b7f1, entries=150, sequenceid=369, filesize=12.0 K 2024-12-08T08:14:26,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/85045a8ec5fd481abb4277383c52fbe2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/85045a8ec5fd481abb4277383c52fbe2 2024-12-08T08:14:26,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/85045a8ec5fd481abb4277383c52fbe2, entries=150, sequenceid=369, filesize=12.0 K 2024-12-08T08:14:26,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 882ms, sequenceid=369, compaction requested=true 2024-12-08T08:14:26,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:26,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:26,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:26,124 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:26,124 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:26,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:26,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:26,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:26,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:26,125 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:26,125 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/A is initiating minor compaction (all files) 2024-12-08T08:14:26,126 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/A in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:26,126 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/29bbcc7ab34045a89434e184ffd847cd, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/8c46649d2094471c844e22136e15cc75, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/0bbb817bfe194166a5ddfe42d0534dd8] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=39.3 K 2024-12-08T08:14:26,126 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:26,126 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/B is initiating minor compaction (all files) 2024-12-08T08:14:26,126 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/B in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:26,126 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29bbcc7ab34045a89434e184ffd847cd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733645663429 2024-12-08T08:14:26,126 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/76b6c802d7d843e1ad69e76ae54a041b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/2b4d497153d04006a3c6981d8e4df9d3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/8beb8e7818804300959cf4959624b7f1] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=36.9 K 2024-12-08T08:14:26,126 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c46649d2094471c844e22136e15cc75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733645664087 2024-12-08T08:14:26,127 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 76b6c802d7d843e1ad69e76ae54a041b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733645663429 2024-12-08T08:14:26,127 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bbb817bfe194166a5ddfe42d0534dd8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733645664724 2024-12-08T08:14:26,127 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b4d497153d04006a3c6981d8e4df9d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733645664087 2024-12-08T08:14:26,127 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 8beb8e7818804300959cf4959624b7f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733645664724 2024-12-08T08:14:26,136 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:26,137 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/91a9638bade941b1b11fcc7890fbeb61 is 50, key is test_row_0/B:col10/1733645665241/Put/seqid=0 2024-12-08T08:14:26,138 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:26,138 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/ee9bc0e3ffc44b31ae7dc0361b3fc13f is 50, key is test_row_0/A:col10/1733645665241/Put/seqid=0 2024-12-08T08:14:26,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741925_1101 (size=13289) 2024-12-08T08:14:26,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741924_1100 (size=13289) 2024-12-08T08:14:26,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T08:14:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:26,178 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T08:14:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:26,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/d2485a9a875746bca71d2682f06f31d3 is 50, key is test_row_0/A:col10/1733645665284/Put/seqid=0 2024-12-08T08:14:26,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741926_1102 (size=12301) 2024-12-08T08:14:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T08:14:26,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. as already flushing 2024-12-08T08:14:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:26,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645726486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:26,555 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/91a9638bade941b1b11fcc7890fbeb61 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/91a9638bade941b1b11fcc7890fbeb61 2024-12-08T08:14:26,555 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/ee9bc0e3ffc44b31ae7dc0361b3fc13f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/ee9bc0e3ffc44b31ae7dc0361b3fc13f 2024-12-08T08:14:26,565 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/A of 6e9f0e742ef7ddcc8f58c1d611d5e17b into ee9bc0e3ffc44b31ae7dc0361b3fc13f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:26,565 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/B of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 91a9638bade941b1b11fcc7890fbeb61(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:26,565 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:26,565 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:26,565 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/B, priority=13, startTime=1733645666124; duration=0sec 2024-12-08T08:14:26,565 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/A, priority=13, startTime=1733645666124; duration=0sec 2024-12-08T08:14:26,565 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:26,565 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:B 2024-12-08T08:14:26,565 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:26,565 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:A 2024-12-08T08:14:26,565 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:26,567 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:26,567 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/C is initiating minor compaction (all files) 2024-12-08T08:14:26,567 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/C in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:26,567 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/b921b2d3a26b4ee082bc603426c94479, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/91404337f69843348a7094c19262dc1c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/85045a8ec5fd481abb4277383c52fbe2] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=36.9 K 2024-12-08T08:14:26,568 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting b921b2d3a26b4ee082bc603426c94479, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733645663429 2024-12-08T08:14:26,568 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 91404337f69843348a7094c19262dc1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733645664087 2024-12-08T08:14:26,572 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 85045a8ec5fd481abb4277383c52fbe2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733645664724 2024-12-08T08:14:26,587 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#C#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:26,588 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/d2485a9a875746bca71d2682f06f31d3 2024-12-08T08:14:26,591 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/f10d5766abf14fd5aeb035ed6ae9ade6 is 50, key is test_row_0/C:col10/1733645665241/Put/seqid=0 2024-12-08T08:14:26,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645726600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:26,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:26,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/f6696584ac8340d0892910772862b9de is 50, key is test_row_0/B:col10/1733645665284/Put/seqid=0 2024-12-08T08:14:26,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741927_1103 (size=13289) 2024-12-08T08:14:26,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741928_1104 (size=12301) 2024-12-08T08:14:26,626 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/f10d5766abf14fd5aeb035ed6ae9ade6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/f10d5766abf14fd5aeb035ed6ae9ade6 2024-12-08T08:14:26,633 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/C of 6e9f0e742ef7ddcc8f58c1d611d5e17b into f10d5766abf14fd5aeb035ed6ae9ade6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:26,634 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:26,634 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/C, priority=13, startTime=1733645666125; duration=0sec 2024-12-08T08:14:26,634 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:26,634 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:C 2024-12-08T08:14:26,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645726810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:26,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40788 deadline: 1733645726891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:26,895 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 6e9f0e742ef7ddcc8f58c1d611d5e17b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:26,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40774 deadline: 1733645726900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:26,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40772 deadline: 1733645726900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:26,905 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 6e9f0e742ef7ddcc8f58c1d611d5e17b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:26,905 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 6e9f0e742ef7ddcc8f58c1d611d5e17b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:27,020 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/f6696584ac8340d0892910772862b9de 2024-12-08T08:14:27,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/e6e0cf4cfc5f473b90b09eed2e40c88b is 50, key is test_row_0/C:col10/1733645665284/Put/seqid=0 2024-12-08T08:14:27,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741929_1105 (size=12301) 2024-12-08T08:14:27,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645727121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T08:14:27,434 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/e6e0cf4cfc5f473b90b09eed2e40c88b 2024-12-08T08:14:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/d2485a9a875746bca71d2682f06f31d3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d2485a9a875746bca71d2682f06f31d3 2024-12-08T08:14:27,445 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d2485a9a875746bca71d2682f06f31d3, entries=150, sequenceid=385, filesize=12.0 K 2024-12-08T08:14:27,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/f6696584ac8340d0892910772862b9de as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/f6696584ac8340d0892910772862b9de 2024-12-08T08:14:27,451 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/f6696584ac8340d0892910772862b9de, entries=150, sequenceid=385, filesize=12.0 K 2024-12-08T08:14:27,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/e6e0cf4cfc5f473b90b09eed2e40c88b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/e6e0cf4cfc5f473b90b09eed2e40c88b 2024-12-08T08:14:27,457 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/e6e0cf4cfc5f473b90b09eed2e40c88b, entries=150, sequenceid=385, filesize=12.0 K 2024-12-08T08:14:27,458 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 1280ms, sequenceid=385, compaction requested=false 2024-12-08T08:14:27,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:27,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:27,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-08T08:14:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-08T08:14:27,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-08T08:14:27,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3550 sec 2024-12-08T08:14:27,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.3600 sec 2024-12-08T08:14:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:27,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T08:14:27,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:27,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:27,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:27,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:27,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:27,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:27,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/e37038bf19ba4afa9ef0bc65ee2e6665 is 50, key is test_row_0/A:col10/1733645667631/Put/seqid=0 2024-12-08T08:14:27,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741930_1106 (size=14741) 2024-12-08T08:14:27,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645727674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:27,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645727780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:27,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:27,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:27,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:27,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645727991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:27,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:27,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:27,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:28,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/e37038bf19ba4afa9ef0bc65ee2e6665 2024-12-08T08:14:28,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/310d5d0bc3634d6895b544ff5b6a550b is 50, key is test_row_0/B:col10/1733645667631/Put/seqid=0 2024-12-08T08:14:28,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741931_1107 (size=12301) 2024-12-08T08:14:28,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645728300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:28,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/310d5d0bc3634d6895b544ff5b6a550b 2024-12-08T08:14:28,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/efbc23d59588444f91bc3c13cc819aad is 50, key is test_row_0/C:col10/1733645667631/Put/seqid=0 2024-12-08T08:14:28,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741932_1108 (size=12301) 2024-12-08T08:14:28,782 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-08T08:14:28,782 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-08T08:14:28,782 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:28,782 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:28,782 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-08T08:14:28,782 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:28,782 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-08T08:14:28,783 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:28,788 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,788 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,788 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,788 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,789 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:28,789 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:28,789 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:28,791 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,791 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:28,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40738 deadline: 1733645728811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:14:28,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/efbc23d59588444f91bc3c13cc819aad 2024-12-08T08:14:28,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/e37038bf19ba4afa9ef0bc65ee2e6665 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e37038bf19ba4afa9ef0bc65ee2e6665 2024-12-08T08:14:28,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e37038bf19ba4afa9ef0bc65ee2e6665, entries=200, sequenceid=405, filesize=14.4 K 2024-12-08T08:14:28,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/310d5d0bc3634d6895b544ff5b6a550b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/310d5d0bc3634d6895b544ff5b6a550b 2024-12-08T08:14:28,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/310d5d0bc3634d6895b544ff5b6a550b, entries=150, sequenceid=405, filesize=12.0 K 2024-12-08T08:14:28,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/efbc23d59588444f91bc3c13cc819aad as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/efbc23d59588444f91bc3c13cc819aad 2024-12-08T08:14:28,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/efbc23d59588444f91bc3c13cc819aad, entries=150, sequenceid=405, filesize=12.0 K 2024-12-08T08:14:28,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 1258ms, sequenceid=405, compaction requested=true 2024-12-08T08:14:28,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:28,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:28,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:28,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:28,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:28,891 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:28,891 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:28,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e9f0e742ef7ddcc8f58c1d611d5e17b:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:28,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:28,892 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:28,892 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:28,892 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/B is initiating minor compaction (all files) 2024-12-08T08:14:28,892 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/A is initiating minor compaction (all files) 2024-12-08T08:14:28,892 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/B in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:28,892 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/A in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:28,892 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/91a9638bade941b1b11fcc7890fbeb61, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/f6696584ac8340d0892910772862b9de, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/310d5d0bc3634d6895b544ff5b6a550b] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=37.0 K 2024-12-08T08:14:28,892 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/ee9bc0e3ffc44b31ae7dc0361b3fc13f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d2485a9a875746bca71d2682f06f31d3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e37038bf19ba4afa9ef0bc65ee2e6665] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=39.4 K 2024-12-08T08:14:28,893 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 91a9638bade941b1b11fcc7890fbeb61, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733645664724 2024-12-08T08:14:28,893 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee9bc0e3ffc44b31ae7dc0361b3fc13f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733645664724 2024-12-08T08:14:28,893 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting f6696584ac8340d0892910772862b9de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733645665284 2024-12-08T08:14:28,893 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2485a9a875746bca71d2682f06f31d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733645665284 2024-12-08T08:14:28,893 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 310d5d0bc3634d6895b544ff5b6a550b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733645666471 2024-12-08T08:14:28,893 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e37038bf19ba4afa9ef0bc65ee2e6665, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733645666471 2024-12-08T08:14:28,907 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#B#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:28,907 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:28,907 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/6f2a0537fda64e45b99ebf2c75dbb3b2 is 50, key is test_row_0/B:col10/1733645667631/Put/seqid=0 2024-12-08T08:14:28,908 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/100347f8d37b4c059714c6da5d515d04 is 50, key is test_row_0/A:col10/1733645667631/Put/seqid=0 2024-12-08T08:14:28,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:28,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T08:14:28,913 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-08T08:14:28,913 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:28,913 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:28,913 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:28,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:28,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:28,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:28,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:28,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:28,914 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-08T08:14:28,914 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:28,914 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,914 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,914 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:28,922 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-08T08:14:28,922 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:28,922 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,922 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:28,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:28,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741933_1109 (size=13391) 2024-12-08T08:14:28,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/1cd278a33a8d4541956a687ade512a48 is 50, key is test_row_0/A:col10/1733645667671/Put/seqid=0 2024-12-08T08:14:28,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741934_1110 (size=13391) 2024-12-08T08:14:28,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741935_1111 (size=12301) 2024-12-08T08:14:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T08:14:29,281 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:29,333 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/6f2a0537fda64e45b99ebf2c75dbb3b2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/6f2a0537fda64e45b99ebf2c75dbb3b2 2024-12-08T08:14:29,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/1cd278a33a8d4541956a687ade512a48 2024-12-08T08:14:29,338 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/100347f8d37b4c059714c6da5d515d04 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/100347f8d37b4c059714c6da5d515d04 2024-12-08T08:14:29,339 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/B of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 6f2a0537fda64e45b99ebf2c75dbb3b2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:29,339 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:29,339 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/B, priority=13, startTime=1733645668891; duration=0sec 2024-12-08T08:14:29,339 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:29,339 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:B 2024-12-08T08:14:29,339 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:29,340 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:29,340 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 6e9f0e742ef7ddcc8f58c1d611d5e17b/C is initiating minor compaction (all files) 2024-12-08T08:14:29,340 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e9f0e742ef7ddcc8f58c1d611d5e17b/C in TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:29,340 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/f10d5766abf14fd5aeb035ed6ae9ade6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/e6e0cf4cfc5f473b90b09eed2e40c88b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/efbc23d59588444f91bc3c13cc819aad] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp, totalSize=37.0 K 2024-12-08T08:14:29,341 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting f10d5766abf14fd5aeb035ed6ae9ade6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733645664724 2024-12-08T08:14:29,341 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting e6e0cf4cfc5f473b90b09eed2e40c88b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733645665284 2024-12-08T08:14:29,341 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting efbc23d59588444f91bc3c13cc819aad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733645666471 2024-12-08T08:14:29,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/1e6bfaed305548b3a2d7a3bccf017167 is 50, key is test_row_0/B:col10/1733645667671/Put/seqid=0 2024-12-08T08:14:29,344 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/A of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 100347f8d37b4c059714c6da5d515d04(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:29,344 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:29,344 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/A, priority=13, startTime=1733645668891; duration=0sec 2024-12-08T08:14:29,344 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:29,344 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:A 2024-12-08T08:14:29,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741936_1112 (size=12301) 2024-12-08T08:14:29,349 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e9f0e742ef7ddcc8f58c1d611d5e17b#C#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:29,349 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/22831f01f3a943249dd4f545b2e7dbb6 is 50, key is test_row_0/C:col10/1733645667631/Put/seqid=0 2024-12-08T08:14:29,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741937_1113 (size=13391) 2024-12-08T08:14:29,482 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-08T08:14:29,482 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:29,482 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:29,482 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:29,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:29,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/1e6bfaed305548b3a2d7a3bccf017167 2024-12-08T08:14:29,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/d7302086c075455b8c8a4adcc78dd579 is 50, key is test_row_0/C:col10/1733645667671/Put/seqid=0 2024-12-08T08:14:29,758 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/22831f01f3a943249dd4f545b2e7dbb6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/22831f01f3a943249dd4f545b2e7dbb6 2024-12-08T08:14:29,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741938_1114 (size=12301) 2024-12-08T08:14:29,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/d7302086c075455b8c8a4adcc78dd579 2024-12-08T08:14:29,768 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e9f0e742ef7ddcc8f58c1d611d5e17b/C of 6e9f0e742ef7ddcc8f58c1d611d5e17b into 22831f01f3a943249dd4f545b2e7dbb6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:29,768 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:29,768 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., storeName=6e9f0e742ef7ddcc8f58c1d611d5e17b/C, priority=13, startTime=1733645668891; duration=0sec 2024-12-08T08:14:29,769 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:29,769 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e9f0e742ef7ddcc8f58c1d611d5e17b:C 2024-12-08T08:14:29,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/1cd278a33a8d4541956a687ade512a48 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1cd278a33a8d4541956a687ade512a48 2024-12-08T08:14:29,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1cd278a33a8d4541956a687ade512a48, entries=150, sequenceid=422, filesize=12.0 K 2024-12-08T08:14:29,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/1e6bfaed305548b3a2d7a3bccf017167 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/1e6bfaed305548b3a2d7a3bccf017167 2024-12-08T08:14:29,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/1e6bfaed305548b3a2d7a3bccf017167, entries=150, sequenceid=422, filesize=12.0 K 2024-12-08T08:14:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/d7302086c075455b8c8a4adcc78dd579 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/d7302086c075455b8c8a4adcc78dd579 2024-12-08T08:14:29,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/d7302086c075455b8c8a4adcc78dd579, entries=150, sequenceid=422, filesize=12.0 K 2024-12-08T08:14:29,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=13.42 KB/13740 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 873ms, sequenceid=422, compaction requested=false 2024-12-08T08:14:29,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: 2024-12-08T08:14:29,824 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-08T08:14:29,824 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:29,824 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:29,824 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:29,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4016 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3946 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1778 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5326 rows 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1778 2024-12-08T08:14:29,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5329 rows 2024-12-08T08:14:29,826 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T08:14:29,826 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T08:14:29,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:29,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:29,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:29,832 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-08T08:14:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T08:14:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T08:14:29,839 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645669839"}]},"ts":"1733645669839"} 2024-12-08T08:14:29,841 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T08:14:29,841 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T08:14:29,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T08:14:29,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e9f0e742ef7ddcc8f58c1d611d5e17b, UNASSIGN}] 2024-12-08T08:14:29,847 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e9f0e742ef7ddcc8f58c1d611d5e17b, UNASSIGN 2024-12-08T08:14:29,848 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6e9f0e742ef7ddcc8f58c1d611d5e17b, regionState=CLOSING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:14:29,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e9f0e742ef7ddcc8f58c1d611d5e17b, UNASSIGN because future has completed 2024-12-08T08:14:29,851 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T08:14:29,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:14:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T08:14:30,007 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:30,007 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T08:14:30,008 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 6e9f0e742ef7ddcc8f58c1d611d5e17b, disabling compactions & flushes 2024-12-08T08:14:30,008 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:30,008 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:30,008 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. after waiting 0 ms 2024-12-08T08:14:30,008 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:30,008 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-08T08:14:30,009 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=A 2024-12-08T08:14:30,009 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:30,009 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=B 2024-12-08T08:14:30,009 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:30,009 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6e9f0e742ef7ddcc8f58c1d611d5e17b, store=C 2024-12-08T08:14:30,009 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:30,014 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/8cac89b448474d908c4edc42c8ac0703 is 50, key is test_row_2/A:col10/1733645669822/Put/seqid=0 2024-12-08T08:14:30,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741939_1115 (size=7415) 2024-12-08T08:14:30,021 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/8cac89b448474d908c4edc42c8ac0703 2024-12-08T08:14:30,030 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/64117bbd6ef44cd5af8eb5380efe8911 is 50, key is test_row_2/B:col10/1733645669822/Put/seqid=0 2024-12-08T08:14:30,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741940_1116 (size=7415) 2024-12-08T08:14:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T08:14:30,406 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T08:14:30,439 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/64117bbd6ef44cd5af8eb5380efe8911 2024-12-08T08:14:30,446 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/a8df0436e3e04791a6b242a5ac227da2 is 50, key is test_row_2/C:col10/1733645669822/Put/seqid=0 2024-12-08T08:14:30,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741941_1117 (size=7415) 2024-12-08T08:14:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T08:14:30,852 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/a8df0436e3e04791a6b242a5ac227da2 2024-12-08T08:14:30,862 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/A/8cac89b448474d908c4edc42c8ac0703 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/8cac89b448474d908c4edc42c8ac0703 2024-12-08T08:14:30,868 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/8cac89b448474d908c4edc42c8ac0703, entries=50, sequenceid=431, filesize=7.2 K 2024-12-08T08:14:30,869 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/B/64117bbd6ef44cd5af8eb5380efe8911 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/64117bbd6ef44cd5af8eb5380efe8911 2024-12-08T08:14:30,874 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/64117bbd6ef44cd5af8eb5380efe8911, entries=50, sequenceid=431, filesize=7.2 K 2024-12-08T08:14:30,875 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/.tmp/C/a8df0436e3e04791a6b242a5ac227da2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/a8df0436e3e04791a6b242a5ac227da2 2024-12-08T08:14:30,879 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/a8df0436e3e04791a6b242a5ac227da2, entries=50, sequenceid=431, filesize=7.2 K 2024-12-08T08:14:30,880 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 872ms, sequenceid=431, compaction requested=true 2024-12-08T08:14:30,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d13a483eb25b4d9688d51b3625e5ba3e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/dfbaf8c57a2040e194ea459dfa1d2007, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/232edbcd4bfb4ab4b35b304a2ce0c0f4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/178048a2c21c479aba5f0f2789d7138f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/c17d66f9ac81410a9d381c4875a0bde0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/25c12ae2fdf247ea8486a2695453bf9c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1541c999376c488a812f6d51df67d404, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/c7381e3f767d4bb9a22782c48d449978, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/18008434e2804efe8216b433f7c83506, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/34a9a4ce644b460387c4328c9314782e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e4b611d78ebf4506a84dc0791f5eac4c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d2d2533d6b9b4aa4874e9b8cebd8060c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e213e29eaee14af095dcb85da3efedcf, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1fafb508a1464fcd88e0b033b6c03c21, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/412353aea9d944a58cf4a7d394cea06e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/29d3eaf1aa604441b68fcb54b8a855b5, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/475a7a9f14f24e3a9f023c5398c8a65d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e36d76dac77341bfa9dd73a39824d21b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/b0cb1a5da44842edb45f6e0d3a55ae36, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/cf776c830cff4befaf04af575f906779, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/6ad2a86b228a4fd7adfc96602a73712e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/3af11c8a0608423095c3b561ee421d6b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/9a34d6daa37c4490a3b1dbd8d73720a1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/4730ca4eed3f4d06a2d0bc1be3530c2a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/be08f410a1ae4a4ab81057de80a888e6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/4897509d03d941e19406a08beb6548b0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/29bbcc7ab34045a89434e184ffd847cd, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/8c46649d2094471c844e22136e15cc75, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/0bbb817bfe194166a5ddfe42d0534dd8, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/ee9bc0e3ffc44b31ae7dc0361b3fc13f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d2485a9a875746bca71d2682f06f31d3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e37038bf19ba4afa9ef0bc65ee2e6665] to archive 2024-12-08T08:14:30,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:14:30,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d13a483eb25b4d9688d51b3625e5ba3e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d13a483eb25b4d9688d51b3625e5ba3e 2024-12-08T08:14:30,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/dfbaf8c57a2040e194ea459dfa1d2007 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/dfbaf8c57a2040e194ea459dfa1d2007 2024-12-08T08:14:30,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/232edbcd4bfb4ab4b35b304a2ce0c0f4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/232edbcd4bfb4ab4b35b304a2ce0c0f4 2024-12-08T08:14:30,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/178048a2c21c479aba5f0f2789d7138f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/178048a2c21c479aba5f0f2789d7138f 2024-12-08T08:14:30,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/c17d66f9ac81410a9d381c4875a0bde0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/c17d66f9ac81410a9d381c4875a0bde0 2024-12-08T08:14:30,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/25c12ae2fdf247ea8486a2695453bf9c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/25c12ae2fdf247ea8486a2695453bf9c 2024-12-08T08:14:30,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1541c999376c488a812f6d51df67d404 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1541c999376c488a812f6d51df67d404 2024-12-08T08:14:30,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/c7381e3f767d4bb9a22782c48d449978 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/c7381e3f767d4bb9a22782c48d449978 2024-12-08T08:14:30,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/18008434e2804efe8216b433f7c83506 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/18008434e2804efe8216b433f7c83506 2024-12-08T08:14:30,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/34a9a4ce644b460387c4328c9314782e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/34a9a4ce644b460387c4328c9314782e 2024-12-08T08:14:30,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e4b611d78ebf4506a84dc0791f5eac4c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e4b611d78ebf4506a84dc0791f5eac4c 2024-12-08T08:14:30,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d2d2533d6b9b4aa4874e9b8cebd8060c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d2d2533d6b9b4aa4874e9b8cebd8060c 2024-12-08T08:14:30,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e213e29eaee14af095dcb85da3efedcf to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e213e29eaee14af095dcb85da3efedcf 2024-12-08T08:14:30,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1fafb508a1464fcd88e0b033b6c03c21 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1fafb508a1464fcd88e0b033b6c03c21 2024-12-08T08:14:30,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/412353aea9d944a58cf4a7d394cea06e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/412353aea9d944a58cf4a7d394cea06e 2024-12-08T08:14:30,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/29d3eaf1aa604441b68fcb54b8a855b5 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/29d3eaf1aa604441b68fcb54b8a855b5 2024-12-08T08:14:30,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/475a7a9f14f24e3a9f023c5398c8a65d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/475a7a9f14f24e3a9f023c5398c8a65d 2024-12-08T08:14:30,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e36d76dac77341bfa9dd73a39824d21b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e36d76dac77341bfa9dd73a39824d21b 2024-12-08T08:14:30,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/b0cb1a5da44842edb45f6e0d3a55ae36 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/b0cb1a5da44842edb45f6e0d3a55ae36 2024-12-08T08:14:30,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/cf776c830cff4befaf04af575f906779 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/cf776c830cff4befaf04af575f906779 2024-12-08T08:14:30,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/6ad2a86b228a4fd7adfc96602a73712e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/6ad2a86b228a4fd7adfc96602a73712e 2024-12-08T08:14:30,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/3af11c8a0608423095c3b561ee421d6b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/3af11c8a0608423095c3b561ee421d6b 2024-12-08T08:14:30,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/9a34d6daa37c4490a3b1dbd8d73720a1 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/9a34d6daa37c4490a3b1dbd8d73720a1 2024-12-08T08:14:30,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/4730ca4eed3f4d06a2d0bc1be3530c2a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/4730ca4eed3f4d06a2d0bc1be3530c2a 2024-12-08T08:14:30,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/be08f410a1ae4a4ab81057de80a888e6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/be08f410a1ae4a4ab81057de80a888e6 2024-12-08T08:14:30,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/4897509d03d941e19406a08beb6548b0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/4897509d03d941e19406a08beb6548b0 2024-12-08T08:14:30,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/29bbcc7ab34045a89434e184ffd847cd to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/29bbcc7ab34045a89434e184ffd847cd 2024-12-08T08:14:30,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/8c46649d2094471c844e22136e15cc75 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/8c46649d2094471c844e22136e15cc75 2024-12-08T08:14:30,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/0bbb817bfe194166a5ddfe42d0534dd8 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/0bbb817bfe194166a5ddfe42d0534dd8 2024-12-08T08:14:30,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/ee9bc0e3ffc44b31ae7dc0361b3fc13f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/ee9bc0e3ffc44b31ae7dc0361b3fc13f 2024-12-08T08:14:30,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d2485a9a875746bca71d2682f06f31d3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/d2485a9a875746bca71d2682f06f31d3 2024-12-08T08:14:30,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e37038bf19ba4afa9ef0bc65ee2e6665 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/e37038bf19ba4afa9ef0bc65ee2e6665 2024-12-08T08:14:30,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/939c36cd24c9471cb815f91c07bc94be, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/cac01e61271d499d88bcca26b939df90, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/5e2d75b9ddbc4fd496e160f95b6f8c11, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/aeb14fcea3c348689c484194d65241eb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/2635cc1541a24e8e947729e00ea8293d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/a15518b5f499405c80e3c28f4c953b0d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/777f481474e64c64948f73012879ef51, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/d01573d1a7b34431a79b721c2bb98329, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/85d8eab801ee4eee861807b922a3e863, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/0e065b658536402aa70a276873ba6c31, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/34d919a7ab974c68b746e34817306d52, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/d4418f4f64c6438fb184de2a7c9edb6d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/45b694078bae44af94465c6dcde2abab, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/7250125fc58640fe91572ce17efefe3c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/54288e5084a7469eaf340fa50022f04e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/b40f16a6af6045c598d75a141d42894e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/bcb820fba84846918ef063f974910115, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/807a7c0f5b654ddda8517d28686df137, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/309bf5ef0b344817b11c935307a945e2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/fb1b7df0dde4433c94d65d97e4bf3b98, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/9c011e8788c7472c950bb1c28bbedf01, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/606a178f4b17411499d215925662c1ce, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/63890a71f9c04be1b146ba7cf130d3fc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/3a545c208797480a8cceb403ebe99d08, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/c6dc46728b21428d968d9a3b6309224c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/76b6c802d7d843e1ad69e76ae54a041b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/7fadc42c065a41f5b749ecce2f8b4eba, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/2b4d497153d04006a3c6981d8e4df9d3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/91a9638bade941b1b11fcc7890fbeb61, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/8beb8e7818804300959cf4959624b7f1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/f6696584ac8340d0892910772862b9de, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/310d5d0bc3634d6895b544ff5b6a550b] to archive 2024-12-08T08:14:30,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:14:30,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/939c36cd24c9471cb815f91c07bc94be to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/939c36cd24c9471cb815f91c07bc94be 2024-12-08T08:14:30,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/cac01e61271d499d88bcca26b939df90 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/cac01e61271d499d88bcca26b939df90 2024-12-08T08:14:30,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/5e2d75b9ddbc4fd496e160f95b6f8c11 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/5e2d75b9ddbc4fd496e160f95b6f8c11 2024-12-08T08:14:30,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/aeb14fcea3c348689c484194d65241eb to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/aeb14fcea3c348689c484194d65241eb 2024-12-08T08:14:30,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/2635cc1541a24e8e947729e00ea8293d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/2635cc1541a24e8e947729e00ea8293d 2024-12-08T08:14:30,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/a15518b5f499405c80e3c28f4c953b0d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/a15518b5f499405c80e3c28f4c953b0d 2024-12-08T08:14:30,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/777f481474e64c64948f73012879ef51 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/777f481474e64c64948f73012879ef51 2024-12-08T08:14:30,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/d01573d1a7b34431a79b721c2bb98329 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/d01573d1a7b34431a79b721c2bb98329 2024-12-08T08:14:30,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/85d8eab801ee4eee861807b922a3e863 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/85d8eab801ee4eee861807b922a3e863 2024-12-08T08:14:30,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/0e065b658536402aa70a276873ba6c31 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/0e065b658536402aa70a276873ba6c31 2024-12-08T08:14:30,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/34d919a7ab974c68b746e34817306d52 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/34d919a7ab974c68b746e34817306d52 2024-12-08T08:14:30,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/d4418f4f64c6438fb184de2a7c9edb6d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/d4418f4f64c6438fb184de2a7c9edb6d 2024-12-08T08:14:30,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/45b694078bae44af94465c6dcde2abab to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/45b694078bae44af94465c6dcde2abab 2024-12-08T08:14:30,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/7250125fc58640fe91572ce17efefe3c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/7250125fc58640fe91572ce17efefe3c 2024-12-08T08:14:30,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/54288e5084a7469eaf340fa50022f04e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/54288e5084a7469eaf340fa50022f04e 2024-12-08T08:14:30,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/b40f16a6af6045c598d75a141d42894e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/b40f16a6af6045c598d75a141d42894e 2024-12-08T08:14:30,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/bcb820fba84846918ef063f974910115 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/bcb820fba84846918ef063f974910115 2024-12-08T08:14:30,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/807a7c0f5b654ddda8517d28686df137 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/807a7c0f5b654ddda8517d28686df137 2024-12-08T08:14:30,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/309bf5ef0b344817b11c935307a945e2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/309bf5ef0b344817b11c935307a945e2 2024-12-08T08:14:30,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/fb1b7df0dde4433c94d65d97e4bf3b98 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/fb1b7df0dde4433c94d65d97e4bf3b98 2024-12-08T08:14:30,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/9c011e8788c7472c950bb1c28bbedf01 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/9c011e8788c7472c950bb1c28bbedf01 2024-12-08T08:14:30,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/606a178f4b17411499d215925662c1ce to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/606a178f4b17411499d215925662c1ce 2024-12-08T08:14:30,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/63890a71f9c04be1b146ba7cf130d3fc to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/63890a71f9c04be1b146ba7cf130d3fc 2024-12-08T08:14:30,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/3a545c208797480a8cceb403ebe99d08 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/3a545c208797480a8cceb403ebe99d08 2024-12-08T08:14:30,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/c6dc46728b21428d968d9a3b6309224c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/c6dc46728b21428d968d9a3b6309224c 2024-12-08T08:14:30,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/76b6c802d7d843e1ad69e76ae54a041b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/76b6c802d7d843e1ad69e76ae54a041b 2024-12-08T08:14:30,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/7fadc42c065a41f5b749ecce2f8b4eba to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/7fadc42c065a41f5b749ecce2f8b4eba 2024-12-08T08:14:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T08:14:30,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/2b4d497153d04006a3c6981d8e4df9d3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/2b4d497153d04006a3c6981d8e4df9d3 2024-12-08T08:14:30,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/91a9638bade941b1b11fcc7890fbeb61 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/91a9638bade941b1b11fcc7890fbeb61 2024-12-08T08:14:30,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/8beb8e7818804300959cf4959624b7f1 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/8beb8e7818804300959cf4959624b7f1 2024-12-08T08:14:30,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/f6696584ac8340d0892910772862b9de to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/f6696584ac8340d0892910772862b9de 2024-12-08T08:14:30,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/310d5d0bc3634d6895b544ff5b6a550b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/310d5d0bc3634d6895b544ff5b6a550b 2024-12-08T08:14:30,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/99a515751188472da396dfce9a45f579, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/6f2754910a3249eebdf2c936ed0e8b01, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/45c50450d9f64529b92104469569e147, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/4b890577ad724a01b827595f163f6550, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/2306e80e886a4ad9af0e7d019f93c8bb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/361be88b9ff5499482d9a8521accbe7a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/952a1692537a427dbb63e985cd9d45d2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/02a394e105014430bb3b2e0dd214da8b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/1f17ec22b68c4748b43ac9b28041e37c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/224983b61d0c412dbf98e71c0b80cbb6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/de5068abaead4de0bd7050f7a2d2a037, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/193f9b6fc70d410a984e39feaaed363a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/5a666a9380c74b26a1db15d0bea1ffc9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/8029b92cbf84453cbf5265cdb49e3d08, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/3a8543a73f92435ab6789b0da7465a00, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/531869fa473a4acf824f30a1fa957ca2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/7993062e0c894ea7acb33b7c4baca106, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/d479e95524e548fb8b949ae3cb702d47, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/64200a2eb0e7476caa53694e4ebf6850, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/16028823196746e8904b9c794be981b1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/6881ca2e3ad04648bc671403dcac40f0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/f1eb01f3ef5a433793dca103a19a70f3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/bb5881323453472b868474515b73c9f9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/cbf63a59d4a74a69acdf428ddb726a76, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/1a9f1a71688748eda3cb7b2157007cca, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/b921b2d3a26b4ee082bc603426c94479, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/186fcbf4f64647b5bbbbbdb833b80160, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/91404337f69843348a7094c19262dc1c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/f10d5766abf14fd5aeb035ed6ae9ade6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/85045a8ec5fd481abb4277383c52fbe2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/e6e0cf4cfc5f473b90b09eed2e40c88b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/efbc23d59588444f91bc3c13cc819aad] to archive 2024-12-08T08:14:30,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:14:30,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/99a515751188472da396dfce9a45f579 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/99a515751188472da396dfce9a45f579 2024-12-08T08:14:30,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/6f2754910a3249eebdf2c936ed0e8b01 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/6f2754910a3249eebdf2c936ed0e8b01 2024-12-08T08:14:30,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/45c50450d9f64529b92104469569e147 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/45c50450d9f64529b92104469569e147 2024-12-08T08:14:30,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/4b890577ad724a01b827595f163f6550 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/4b890577ad724a01b827595f163f6550 2024-12-08T08:14:30,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/2306e80e886a4ad9af0e7d019f93c8bb to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/2306e80e886a4ad9af0e7d019f93c8bb 2024-12-08T08:14:30,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/361be88b9ff5499482d9a8521accbe7a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/361be88b9ff5499482d9a8521accbe7a 2024-12-08T08:14:30,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/952a1692537a427dbb63e985cd9d45d2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/952a1692537a427dbb63e985cd9d45d2 2024-12-08T08:14:30,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/02a394e105014430bb3b2e0dd214da8b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/02a394e105014430bb3b2e0dd214da8b 2024-12-08T08:14:30,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/1f17ec22b68c4748b43ac9b28041e37c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/1f17ec22b68c4748b43ac9b28041e37c 2024-12-08T08:14:30,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/224983b61d0c412dbf98e71c0b80cbb6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/224983b61d0c412dbf98e71c0b80cbb6 2024-12-08T08:14:30,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/de5068abaead4de0bd7050f7a2d2a037 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/de5068abaead4de0bd7050f7a2d2a037 2024-12-08T08:14:30,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/193f9b6fc70d410a984e39feaaed363a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/193f9b6fc70d410a984e39feaaed363a 2024-12-08T08:14:30,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/5a666a9380c74b26a1db15d0bea1ffc9 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/5a666a9380c74b26a1db15d0bea1ffc9 2024-12-08T08:14:30,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/8029b92cbf84453cbf5265cdb49e3d08 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/8029b92cbf84453cbf5265cdb49e3d08 2024-12-08T08:14:30,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/3a8543a73f92435ab6789b0da7465a00 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/3a8543a73f92435ab6789b0da7465a00 2024-12-08T08:14:30,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/531869fa473a4acf824f30a1fa957ca2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/531869fa473a4acf824f30a1fa957ca2 2024-12-08T08:14:30,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/7993062e0c894ea7acb33b7c4baca106 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/7993062e0c894ea7acb33b7c4baca106 2024-12-08T08:14:30,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/d479e95524e548fb8b949ae3cb702d47 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/d479e95524e548fb8b949ae3cb702d47 2024-12-08T08:14:31,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/64200a2eb0e7476caa53694e4ebf6850 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/64200a2eb0e7476caa53694e4ebf6850 2024-12-08T08:14:31,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/16028823196746e8904b9c794be981b1 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/16028823196746e8904b9c794be981b1 2024-12-08T08:14:31,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/6881ca2e3ad04648bc671403dcac40f0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/6881ca2e3ad04648bc671403dcac40f0 2024-12-08T08:14:31,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/f1eb01f3ef5a433793dca103a19a70f3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/f1eb01f3ef5a433793dca103a19a70f3 2024-12-08T08:14:31,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/bb5881323453472b868474515b73c9f9 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/bb5881323453472b868474515b73c9f9 2024-12-08T08:14:31,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/cbf63a59d4a74a69acdf428ddb726a76 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/cbf63a59d4a74a69acdf428ddb726a76 2024-12-08T08:14:31,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/1a9f1a71688748eda3cb7b2157007cca to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/1a9f1a71688748eda3cb7b2157007cca 2024-12-08T08:14:31,007 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/b921b2d3a26b4ee082bc603426c94479 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/b921b2d3a26b4ee082bc603426c94479 2024-12-08T08:14:31,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/186fcbf4f64647b5bbbbbdb833b80160 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/186fcbf4f64647b5bbbbbdb833b80160 2024-12-08T08:14:31,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/91404337f69843348a7094c19262dc1c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/91404337f69843348a7094c19262dc1c 2024-12-08T08:14:31,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/f10d5766abf14fd5aeb035ed6ae9ade6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/f10d5766abf14fd5aeb035ed6ae9ade6 2024-12-08T08:14:31,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/85045a8ec5fd481abb4277383c52fbe2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/85045a8ec5fd481abb4277383c52fbe2 2024-12-08T08:14:31,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/e6e0cf4cfc5f473b90b09eed2e40c88b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/e6e0cf4cfc5f473b90b09eed2e40c88b 2024-12-08T08:14:31,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/efbc23d59588444f91bc3c13cc819aad to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/efbc23d59588444f91bc3c13cc819aad 2024-12-08T08:14:31,017 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/recovered.edits/434.seqid, newMaxSeqId=434, maxSeqId=1 2024-12-08T08:14:31,018 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. 2024-12-08T08:14:31,018 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 6e9f0e742ef7ddcc8f58c1d611d5e17b: Waiting for close lock at 1733645670008Running coprocessor pre-close hooks at 1733645670008Disabling compacts and flushes for region at 1733645670008Disabling writes for close at 1733645670008Obtaining lock to block concurrent updates at 1733645670008Preparing flush snapshotting stores in 6e9f0e742ef7ddcc8f58c1d611d5e17b at 1733645670008Finished memstore snapshotting TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733645670009 (+1 ms)Flushing stores of TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b. at 1733645670010 (+1 ms)Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b/A: creating writer at 1733645670010Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b/A: appending metadata at 1733645670013 (+3 ms)Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b/A: closing flushed file at 1733645670013Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b/B: creating writer at 1733645670026 (+13 ms)Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b/B: appending metadata at 1733645670029 (+3 ms)Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b/B: closing flushed file at 1733645670030 (+1 ms)Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b/C: creating writer at 1733645670443 (+413 ms)Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b/C: appending metadata at 1733645670446 (+3 ms)Flushing 6e9f0e742ef7ddcc8f58c1d611d5e17b/C: closing flushed file at 1733645670446Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69f9db7d: reopening flushed file at 1733645670861 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e479560: reopening flushed file at 1733645670868 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70b856e0: reopening flushed file at 1733645670874 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 6e9f0e742ef7ddcc8f58c1d611d5e17b in 872ms, sequenceid=431, compaction requested=true at 1733645670880 (+6 ms)Writing region close event to WAL at 1733645671014 (+134 ms)Running coprocessor post-close hooks at 1733645671017 (+3 ms)Closed at 1733645671018 (+1 ms) 2024-12-08T08:14:31,020 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:31,020 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6e9f0e742ef7ddcc8f58c1d611d5e17b, regionState=CLOSED 2024-12-08T08:14:31,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:14:31,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-08T08:14:31,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 6e9f0e742ef7ddcc8f58c1d611d5e17b, server=422e41b75614,38619,1733645641981 in 1.1720 sec 2024-12-08T08:14:31,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-08T08:14:31,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e9f0e742ef7ddcc8f58c1d611d5e17b, UNASSIGN in 1.1780 sec 2024-12-08T08:14:31,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-08T08:14:31,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1840 sec 2024-12-08T08:14:31,028 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645671028"}]},"ts":"1733645671028"} 2024-12-08T08:14:31,030 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T08:14:31,030 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T08:14:31,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1960 sec 2024-12-08T08:14:31,597 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T08:14:31,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T08:14:31,983 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:31,991 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-08T08:14:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:14:31,997 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:14:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T08:14:31,998 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:14:32,003 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:32,007 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/recovered.edits] 2024-12-08T08:14:32,011 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/100347f8d37b4c059714c6da5d515d04 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/100347f8d37b4c059714c6da5d515d04 2024-12-08T08:14:32,013 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1cd278a33a8d4541956a687ade512a48 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/1cd278a33a8d4541956a687ade512a48 2024-12-08T08:14:32,014 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/8cac89b448474d908c4edc42c8ac0703 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/A/8cac89b448474d908c4edc42c8ac0703 2024-12-08T08:14:32,016 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/1e6bfaed305548b3a2d7a3bccf017167 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/1e6bfaed305548b3a2d7a3bccf017167 2024-12-08T08:14:32,018 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/64117bbd6ef44cd5af8eb5380efe8911 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/64117bbd6ef44cd5af8eb5380efe8911 2024-12-08T08:14:32,019 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/6f2a0537fda64e45b99ebf2c75dbb3b2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/B/6f2a0537fda64e45b99ebf2c75dbb3b2 2024-12-08T08:14:32,021 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/22831f01f3a943249dd4f545b2e7dbb6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/22831f01f3a943249dd4f545b2e7dbb6 2024-12-08T08:14:32,022 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/a8df0436e3e04791a6b242a5ac227da2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/a8df0436e3e04791a6b242a5ac227da2 2024-12-08T08:14:32,023 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/d7302086c075455b8c8a4adcc78dd579 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/C/d7302086c075455b8c8a4adcc78dd579 2024-12-08T08:14:32,025 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/recovered.edits/434.seqid to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b/recovered.edits/434.seqid 2024-12-08T08:14:32,025 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/6e9f0e742ef7ddcc8f58c1d611d5e17b 2024-12-08T08:14:32,025 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T08:14:32,030 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:14:32,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-08T08:14:32,038 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T08:14:32,066 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T08:14:32,068 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:14:32,068 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T08:14:32,068 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733645672068"}]},"ts":"9223372036854775807"} 2024-12-08T08:14:32,073 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T08:14:32,073 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6e9f0e742ef7ddcc8f58c1d611d5e17b, NAME => 'TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T08:14:32,073 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T08:14:32,074 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733645672073"}]},"ts":"9223372036854775807"} 2024-12-08T08:14:32,077 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T08:14:32,078 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:14:32,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-12-08T08:14:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T08:14:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T08:14:32,101 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:32,114 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2010449533_22 at /127.0.0.1:42142 [Waiting for operation #255] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1772320635_22 at /127.0.0.1:50108 [Waiting for operation #153] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2010449533_22 at /127.0.0.1:49836 [Waiting for operation #237] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/422e41b75614:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/422e41b75614:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;422e41b75614:38619-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=373 (was 307) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6440 (was 7042) 2024-12-08T08:14:32,122 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=373, ProcessCount=11, AvailableMemoryMB=6439 2024-12-08T08:14:32,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T08:14:32,123 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:14:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T08:14:32,125 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:14:32,125 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:32,125 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-12-08T08:14:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T08:14:32,126 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:14:32,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741942_1118 (size=1061) 2024-12-08T08:14:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T08:14:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T08:14:32,537 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b 2024-12-08T08:14:32,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741943_1119 (size=53) 2024-12-08T08:14:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T08:14:32,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:14:32,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 45f279431c7ebbce8a61010c6b97623a, disabling compactions & flushes 2024-12-08T08:14:32,952 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:32,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:32,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. after waiting 0 ms 2024-12-08T08:14:32,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:32,953 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:32,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 45f279431c7ebbce8a61010c6b97623a: Waiting for close lock at 1733645672952Disabling compacts and flushes for region at 1733645672952Disabling writes for close at 1733645672953 (+1 ms)Writing region close event to WAL at 1733645672953Closed at 1733645672953 2024-12-08T08:14:32,956 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:14:32,957 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733645672956"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733645672956"}]},"ts":"1733645672956"} 2024-12-08T08:14:32,961 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:14:32,963 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:14:32,963 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645672963"}]},"ts":"1733645672963"} 2024-12-08T08:14:32,964 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T08:14:32,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, ASSIGN}] 2024-12-08T08:14:32,966 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, ASSIGN 2024-12-08T08:14:32,967 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, ASSIGN; state=OFFLINE, location=422e41b75614,38619,1733645641981; forceNewPlan=false, retain=false 2024-12-08T08:14:33,117 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=45f279431c7ebbce8a61010c6b97623a, regionState=OPENING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:14:33,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, ASSIGN because future has completed 2024-12-08T08:14:33,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:14:33,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T08:14:33,279 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:33,279 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:14:33,280 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:33,280 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:14:33,281 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:33,281 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:33,284 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:33,285 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:14:33,286 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45f279431c7ebbce8a61010c6b97623a columnFamilyName A 2024-12-08T08:14:33,286 DEBUG [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:33,286 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.HStore(327): Store=45f279431c7ebbce8a61010c6b97623a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:14:33,286 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:33,287 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:14:33,288 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45f279431c7ebbce8a61010c6b97623a columnFamilyName B 2024-12-08T08:14:33,288 DEBUG [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:33,288 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.HStore(327): Store=45f279431c7ebbce8a61010c6b97623a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:14:33,288 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:33,289 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:14:33,289 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45f279431c7ebbce8a61010c6b97623a columnFamilyName C 2024-12-08T08:14:33,289 DEBUG [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:33,289 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.HStore(327): Store=45f279431c7ebbce8a61010c6b97623a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:14:33,290 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:33,290 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:33,290 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:33,291 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:33,291 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:33,292 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:33,292 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T08:14:33,293 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:33,295 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:14:33,296 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 45f279431c7ebbce8a61010c6b97623a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69102732, jitterRate=0.02971094846725464}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T08:14:33,296 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:33,297 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 45f279431c7ebbce8a61010c6b97623a: Running coprocessor pre-open hook at 1733645673282Writing region info on filesystem at 1733645673282Initializing all the Stores at 1733645673284 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645673284Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645673284Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645673284Cleaning up temporary data from old regions at 1733645673292 (+8 ms)Running coprocessor post-open hooks at 1733645673296 (+4 ms)Region opened successfully at 1733645673297 (+1 ms) 2024-12-08T08:14:33,298 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., pid=34, masterSystemTime=1733645673271 2024-12-08T08:14:33,300 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:33,300 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:33,301 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=45f279431c7ebbce8a61010c6b97623a, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:14:33,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:14:33,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-08T08:14:33,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 in 183 msec 2024-12-08T08:14:33,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-08T08:14:33,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, ASSIGN in 339 msec 2024-12-08T08:14:33,307 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:14:33,307 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645673307"}]},"ts":"1733645673307"} 2024-12-08T08:14:33,309 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T08:14:33,310 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:14:33,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-12-08T08:14:34,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T08:14:34,272 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:34,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a00013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:34,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:34,274 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:34,277 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e528e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:34,279 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:34,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230c8e56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:34,280 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:34,282 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:34,284 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T08:14:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 422e41b75614,44803,1733645641278 2024-12-08T08:14:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5d9da2d8 2024-12-08T08:14:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T08:14:34,286 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T08:14:34,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T08:14:34,292 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:14:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T08:14:34,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741944_1120 (size=1097) 2024-12-08T08:14:34,715 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T08:14:34,715 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T08:14:34,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T08:14:34,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, REOPEN/MOVE}] 2024-12-08T08:14:34,730 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, REOPEN/MOVE 2024-12-08T08:14:34,731 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=45f279431c7ebbce8a61010c6b97623a, regionState=CLOSING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:14:34,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, REOPEN/MOVE because future has completed 2024-12-08T08:14:34,733 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T08:14:34,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:14:34,885 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:34,885 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T08:14:34,885 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 45f279431c7ebbce8a61010c6b97623a, disabling compactions & flushes 2024-12-08T08:14:34,885 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:34,885 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:34,885 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. after waiting 0 ms 2024-12-08T08:14:34,885 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:34,891 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T08:14:34,892 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:34,892 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 45f279431c7ebbce8a61010c6b97623a: Waiting for close lock at 1733645674885Running coprocessor pre-close hooks at 1733645674885Disabling compacts and flushes for region at 1733645674885Disabling writes for close at 1733645674885Writing region close event to WAL at 1733645674886 (+1 ms)Running coprocessor post-close hooks at 1733645674892 (+6 ms)Closed at 1733645674892 2024-12-08T08:14:34,892 WARN [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 45f279431c7ebbce8a61010c6b97623a to self. 2024-12-08T08:14:34,895 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:34,896 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=45f279431c7ebbce8a61010c6b97623a, regionState=CLOSED 2024-12-08T08:14:34,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:14:34,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-08T08:14:34,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 in 166 msec 2024-12-08T08:14:34,901 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, REOPEN/MOVE; state=CLOSED, location=422e41b75614,38619,1733645641981; forceNewPlan=false, retain=true 2024-12-08T08:14:35,052 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=45f279431c7ebbce8a61010c6b97623a, regionState=OPENING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, REOPEN/MOVE because future has completed 2024-12-08T08:14:35,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:14:35,220 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:35,220 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:14:35,221 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,221 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:14:35,221 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,221 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,227 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,228 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:14:35,233 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45f279431c7ebbce8a61010c6b97623a columnFamilyName A 2024-12-08T08:14:35,235 DEBUG [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:35,235 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.HStore(327): Store=45f279431c7ebbce8a61010c6b97623a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:14:35,236 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,236 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:14:35,236 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45f279431c7ebbce8a61010c6b97623a columnFamilyName B 2024-12-08T08:14:35,236 DEBUG [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:35,237 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.HStore(327): Store=45f279431c7ebbce8a61010c6b97623a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:14:35,237 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,237 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:14:35,238 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45f279431c7ebbce8a61010c6b97623a columnFamilyName C 2024-12-08T08:14:35,238 DEBUG [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:35,238 INFO [StoreOpener-45f279431c7ebbce8a61010c6b97623a-1 {}] regionserver.HStore(327): Store=45f279431c7ebbce8a61010c6b97623a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:14:35,238 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:35,238 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,239 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,240 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,241 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,241 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,241 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T08:14:35,242 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,243 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 45f279431c7ebbce8a61010c6b97623a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66942812, jitterRate=-0.002474367618560791}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T08:14:35,243 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,245 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 45f279431c7ebbce8a61010c6b97623a: Running coprocessor pre-open hook at 1733645675221Writing region info on filesystem at 1733645675221Initializing all the Stores at 1733645675223 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645675223Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645675223Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645675223Cleaning up temporary data from old regions at 1733645675241 (+18 ms)Running coprocessor post-open hooks at 1733645675243 (+2 ms)Region opened successfully at 1733645675245 (+2 ms) 2024-12-08T08:14:35,246 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., pid=39, masterSystemTime=1733645675212 2024-12-08T08:14:35,248 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:35,248 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:35,249 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=45f279431c7ebbce8a61010c6b97623a, regionState=OPEN, openSeqNum=5, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:14:35,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-12-08T08:14:35,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 in 193 msec 2024-12-08T08:14:35,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-08T08:14:35,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, REOPEN/MOVE in 524 msec 2024-12-08T08:14:35,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-08T08:14:35,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 534 msec 2024-12-08T08:14:35,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 964 msec 2024-12-08T08:14:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-08T08:14:35,264 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:35,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad8b864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:35,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:35,265 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39dada81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:35,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:35,267 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:35,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5298f036, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:35,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:35,269 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12dd14a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:35,270 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f653c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:35,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:35,272 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7effa9db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:35,274 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6de84ed4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,275 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:35,275 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:35,276 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1179b095, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:35,277 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25a597fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,278 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:35,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:35,279 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b14cb85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:35,281 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b7406a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,283 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:35,283 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:35,284 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ff9012, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:35,285 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4fba57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:35,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:35,287 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a39d9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:35,288 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:35,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e4b293, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:35,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:35,289 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:35,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@691e4cdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:35,290 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292c9d41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,291 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:14:35,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:14:35,292 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:14:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:14:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:14:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b9908f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:14:35,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:14:35,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:35,293 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:14:35,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11636416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:14:35,295 DEBUG [Thread-617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:35,295 DEBUG [Thread-620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:35,295 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:35,295 DEBUG [Thread-609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:35,295 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:35,295 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:35,296 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:35,296 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:35,296 DEBUG [Thread-609 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:35,296 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:35,297 DEBUG [Thread-622 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:35,297 DEBUG [Thread-609 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:35,297 DEBUG [Thread-622 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-08T08:14:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:35,297 DEBUG [Thread-615 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:35,298 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:14:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:35,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T08:14:35,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:35,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:35,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:35,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:35,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:35,300 DEBUG [Thread-615 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:14:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:14:35,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:35,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:35,301 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:35,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:14:35,302 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:35,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:14:35,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:14:35,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:14:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:14:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:14:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:14:35,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:14:35,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:14:35,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:14:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:14:35,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:35,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:35,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:35,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:35,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:35,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:35,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208aa0e58770ef742bb86089522474d6156_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645675314/Put/seqid=0 2024-12-08T08:14:35,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741945_1121 (size=14594) 2024-12-08T08:14:35,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645735367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645735374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,388 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:35,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645675362, max timestamp=1733645675381Num uniques 859; 2024-12-08T08:14:35,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733645675362, max timestamp=1733645675379Num uniques 859; 2024-12-08T08:14:35,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:14:35,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T08:14:35,396 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208aa0e58770ef742bb86089522474d6156_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208aa0e58770ef742bb86089522474d6156_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,398 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/e521e65d8aec4f4b8ced26758a426883, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:35,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=156, cellSize=7140, totalHeapSize=16464, min timestamp=1733645675362, max timestamp=1733645675381Num uniques 859; 2024-12-08T08:14:35,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7140, suffix heap size=117064,new segment heap size=16464  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=156 2024-12-08T08:14:35,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/e521e65d8aec4f4b8ced26758a426883 is 175, key is test_row_0/A:col10/1733645675314/Put/seqid=0 2024-12-08T08:14:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T08:14:35,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741946_1122 (size=39549) 2024-12-08T08:14:35,447 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/e521e65d8aec4f4b8ced26758a426883 2024-12-08T08:14:35,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T08:14:35,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:35,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:35,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645735453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42852 deadline: 1733645735465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645735466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/ae119dfbf25e4808bfdbc3c1595f8c2e is 50, key is test_row_0/B:col10/1733645675314/Put/seqid=0 2024-12-08T08:14:35,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645735491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645735491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741947_1123 (size=12001) 2024-12-08T08:14:35,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/ae119dfbf25e4808bfdbc3c1595f8c2e 2024-12-08T08:14:35,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/b0a04805ed8e440b8a5e6085271305cb is 50, key is test_row_0/C:col10/1733645675314/Put/seqid=0 2024-12-08T08:14:35,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741948_1124 (size=12001) 2024-12-08T08:14:35,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/b0a04805ed8e440b8a5e6085271305cb 2024-12-08T08:14:35,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/e521e65d8aec4f4b8ced26758a426883 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e521e65d8aec4f4b8ced26758a426883 2024-12-08T08:14:35,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645735570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e521e65d8aec4f4b8ced26758a426883, entries=200, sequenceid=15, filesize=38.6 K 2024-12-08T08:14:35,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/ae119dfbf25e4808bfdbc3c1595f8c2e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ae119dfbf25e4808bfdbc3c1595f8c2e 2024-12-08T08:14:35,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ae119dfbf25e4808bfdbc3c1595f8c2e, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T08:14:35,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/b0a04805ed8e440b8a5e6085271305cb as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/b0a04805ed8e440b8a5e6085271305cb 2024-12-08T08:14:35,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42852 deadline: 1733645735581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/b0a04805ed8e440b8a5e6085271305cb, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T08:14:35,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=133.65 KB/136860 for 45f279431c7ebbce8a61010c6b97623a in 281ms, sequenceid=15, compaction requested=false 2024-12-08T08:14:35,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:35,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T08:14:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:35,614 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=133.65 KB heapSize=384.47 KB 2024-12-08T08:14:35,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733645675438, max timestamp=1733645675607Num uniques 1009; 2024-12-08T08:14:35,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-08T08:14:35,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733645675438, max timestamp=1733645675607Num uniques 1018; 2024-12-08T08:14:35,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:35,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T08:14:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T08:14:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T08:14:35,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120888252ff0a2404da08db085b94103033c_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645675438/Put/seqid=0 2024-12-08T08:14:35,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741949_1125 (size=14594) 2024-12-08T08:14:35,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:35,653 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120888252ff0a2404da08db085b94103033c_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120888252ff0a2404da08db085b94103033c_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:35,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/1c79e4b3037e4762af97042fd2750434, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:35,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/1c79e4b3037e4762af97042fd2750434 is 175, key is test_row_0/A:col10/1733645675438/Put/seqid=0 2024-12-08T08:14:35,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741950_1126 (size=39549) 2024-12-08T08:14:35,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645735729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645735739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645735741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645735781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42852 deadline: 1733645735801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645735851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645735853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645735860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T08:14:36,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645736061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645736061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,077 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/1c79e4b3037e4762af97042fd2750434 2024-12-08T08:14:36,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645736081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/b36b96e71e7a4111809d1331ce9182f1 is 50, key is test_row_0/B:col10/1733645675438/Put/seqid=0 2024-12-08T08:14:36,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645736091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741951_1127 (size=12001) 2024-12-08T08:14:36,109 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/b36b96e71e7a4111809d1331ce9182f1 2024-12-08T08:14:36,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/573f0a9575384b7cb07ba3c726bb03d5 is 50, key is test_row_0/C:col10/1733645675438/Put/seqid=0 2024-12-08T08:14:36,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42852 deadline: 1733645736121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741952_1128 (size=12001) 2024-12-08T08:14:36,143 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.83 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/573f0a9575384b7cb07ba3c726bb03d5 2024-12-08T08:14:36,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/1c79e4b3037e4762af97042fd2750434 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1c79e4b3037e4762af97042fd2750434 2024-12-08T08:14:36,155 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1c79e4b3037e4762af97042fd2750434, entries=200, sequenceid=53, filesize=38.6 K 2024-12-08T08:14:36,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/b36b96e71e7a4111809d1331ce9182f1 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/b36b96e71e7a4111809d1331ce9182f1 2024-12-08T08:14:36,164 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/b36b96e71e7a4111809d1331ce9182f1, entries=150, sequenceid=53, filesize=11.7 K 2024-12-08T08:14:36,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/573f0a9575384b7cb07ba3c726bb03d5 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/573f0a9575384b7cb07ba3c726bb03d5 2024-12-08T08:14:36,171 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/573f0a9575384b7cb07ba3c726bb03d5, entries=150, sequenceid=53, filesize=11.7 K 2024-12-08T08:14:36,172 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~64.72 KB/66275, heapSize ~178.45 KB/182728, currentSize=114.05 KB/116790 for 45f279431c7ebbce8a61010c6b97623a in 559ms, sequenceid=53, compaction requested=false 2024-12-08T08:14:36,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:36,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:36,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-08T08:14:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-08T08:14:36,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-08T08:14:36,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 872 msec 2024-12-08T08:14:36,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 882 msec 2024-12-08T08:14:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:36,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-08T08:14:36,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:36,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645675722, max timestamp=1733645676372Num uniques 859; 2024-12-08T08:14:36,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:36,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:36,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645675722, max timestamp=1733645676372Num uniques 859; 2024-12-08T08:14:36,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:14:36,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:36,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:36,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:36,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:14:36,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208155be74d2d604e2ea5a60554d6d5c40a_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645676372/Put/seqid=0 2024-12-08T08:14:36,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741953_1129 (size=17034) 2024-12-08T08:14:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T08:14:36,441 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:36,442 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-08T08:14:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T08:14:36,444 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:36,445 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:36,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:36,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645736455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645736460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645736460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T08:14:36,553 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T08:14:36,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645736571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645736581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645736581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645736600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T08:14:36,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:36,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:36,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:36,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:36,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:36,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42852 deadline: 1733645736631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T08:14:36,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:36,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:36,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:36,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:36,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T08:14:36,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645736792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645736792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645736793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:36,800 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:36,806 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208155be74d2d604e2ea5a60554d6d5c40a_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208155be74d2d604e2ea5a60554d6d5c40a_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:36,809 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/c6c46db862e6431a96d2aed8f19606a3, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:36,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/c6c46db862e6431a96d2aed8f19606a3 is 175, key is test_row_0/A:col10/1733645676372/Put/seqid=0 2024-12-08T08:14:36,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741954_1130 (size=48139) 2024-12-08T08:14:36,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T08:14:36,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:36,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:36,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:36,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:36,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:36,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:37,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T08:14:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:37,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T08:14:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645737100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645737100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645737100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T08:14:37,220 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=75, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/c6c46db862e6431a96d2aed8f19606a3 2024-12-08T08:14:37,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:37,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:37,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:37,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:37,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:37,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/aceabf0b310b4d06b86acb66b1cc6f92 is 50, key is test_row_0/B:col10/1733645676372/Put/seqid=0 2024-12-08T08:14:37,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741955_1131 (size=12001) 2024-12-08T08:14:37,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/aceabf0b310b4d06b86acb66b1cc6f92 2024-12-08T08:14:37,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/a90454cabefa4797ad5ea8a12e223df1 is 50, key is test_row_0/C:col10/1733645676372/Put/seqid=0 2024-12-08T08:14:37,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741956_1132 (size=12001) 2024-12-08T08:14:37,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/a90454cabefa4797ad5ea8a12e223df1 2024-12-08T08:14:37,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/c6c46db862e6431a96d2aed8f19606a3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/c6c46db862e6431a96d2aed8f19606a3 2024-12-08T08:14:37,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/c6c46db862e6431a96d2aed8f19606a3, entries=250, sequenceid=75, filesize=47.0 K 2024-12-08T08:14:37,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/aceabf0b310b4d06b86acb66b1cc6f92 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/aceabf0b310b4d06b86acb66b1cc6f92 2024-12-08T08:14:37,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/aceabf0b310b4d06b86acb66b1cc6f92, entries=150, sequenceid=75, filesize=11.7 K 2024-12-08T08:14:37,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/a90454cabefa4797ad5ea8a12e223df1 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/a90454cabefa4797ad5ea8a12e223df1 2024-12-08T08:14:37,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/a90454cabefa4797ad5ea8a12e223df1, entries=150, sequenceid=75, filesize=11.7 K 2024-12-08T08:14:37,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 45f279431c7ebbce8a61010c6b97623a in 956ms, sequenceid=75, compaction requested=true 2024-12-08T08:14:37,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:37,333 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:37,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:37,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:37,335 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:37,335 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:37,335 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/A is initiating minor compaction (all files) 2024-12-08T08:14:37,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:37,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:37,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:37,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:37,335 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/A in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:37,335 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e521e65d8aec4f4b8ced26758a426883, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1c79e4b3037e4762af97042fd2750434, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/c6c46db862e6431a96d2aed8f19606a3] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=124.3 K 2024-12-08T08:14:37,335 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:37,336 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e521e65d8aec4f4b8ced26758a426883, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1c79e4b3037e4762af97042fd2750434, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/c6c46db862e6431a96d2aed8f19606a3] 2024-12-08T08:14:37,336 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:37,336 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/B is initiating minor compaction (all files) 2024-12-08T08:14:37,336 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/B in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:37,336 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ae119dfbf25e4808bfdbc3c1595f8c2e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/b36b96e71e7a4111809d1331ce9182f1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/aceabf0b310b4d06b86acb66b1cc6f92] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=35.2 K 2024-12-08T08:14:37,337 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e521e65d8aec4f4b8ced26758a426883, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733645675306 2024-12-08T08:14:37,337 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting ae119dfbf25e4808bfdbc3c1595f8c2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733645675306 2024-12-08T08:14:37,337 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting b36b96e71e7a4111809d1331ce9182f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733645675438 2024-12-08T08:14:37,337 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c79e4b3037e4762af97042fd2750434, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733645675438 2024-12-08T08:14:37,338 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6c46db862e6431a96d2aed8f19606a3, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733645675722 2024-12-08T08:14:37,338 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting aceabf0b310b4d06b86acb66b1cc6f92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733645675734 2024-12-08T08:14:37,357 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#B#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:37,358 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/ba9a756a6b2b4fbab26a373c8df95ad2 is 50, key is test_row_0/B:col10/1733645676372/Put/seqid=0 2024-12-08T08:14:37,362 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:37,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741957_1133 (size=12104) 2024-12-08T08:14:37,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T08:14:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:37,381 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T08:14:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:37,390 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412081429ecd0be574c92af616921abb10879_45f279431c7ebbce8a61010c6b97623a store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:37,390 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/ba9a756a6b2b4fbab26a373c8df95ad2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ba9a756a6b2b4fbab26a373c8df95ad2 2024-12-08T08:14:37,412 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412081429ecd0be574c92af616921abb10879_45f279431c7ebbce8a61010c6b97623a, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:37,412 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081429ecd0be574c92af616921abb10879_45f279431c7ebbce8a61010c6b97623a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:37,419 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/B of 45f279431c7ebbce8a61010c6b97623a into ba9a756a6b2b4fbab26a373c8df95ad2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:37,419 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:37,419 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/B, priority=13, startTime=1733645677335; duration=0sec 2024-12-08T08:14:37,419 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:37,419 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:B 2024-12-08T08:14:37,419 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:37,421 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:37,421 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/C is initiating minor compaction (all files) 2024-12-08T08:14:37,421 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/C in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:37,421 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/b0a04805ed8e440b8a5e6085271305cb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/573f0a9575384b7cb07ba3c726bb03d5, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/a90454cabefa4797ad5ea8a12e223df1] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=35.2 K 2024-12-08T08:14:37,422 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting b0a04805ed8e440b8a5e6085271305cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733645675306 2024-12-08T08:14:37,423 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 573f0a9575384b7cb07ba3c726bb03d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733645675438 2024-12-08T08:14:37,424 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting a90454cabefa4797ad5ea8a12e223df1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733645675734 2024-12-08T08:14:37,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d42a866fb28f41cdaaebc7332d1ba3b6_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645676457/Put/seqid=0 2024-12-08T08:14:37,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741958_1134 (size=4469) 2024-12-08T08:14:37,451 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#C#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:37,451 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/51e0e25cc4774806a4e4a082d6d9cea2 is 50, key is test_row_0/C:col10/1733645676372/Put/seqid=0 2024-12-08T08:14:37,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741959_1135 (size=12154) 2024-12-08T08:14:37,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741960_1136 (size=12104) 2024-12-08T08:14:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T08:14:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:37,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:37,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645737623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645737623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645737624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645737629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42852 deadline: 1733645737650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,651 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 45f279431c7ebbce8a61010c6b97623a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:37,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645737741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645737741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645737741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645737741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,849 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#A#compaction#115 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:37,850 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/a0150ee9556a4ee89b50fe7c1f4abe32 is 175, key is test_row_0/A:col10/1733645676372/Put/seqid=0 2024-12-08T08:14:37,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:37,864 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d42a866fb28f41cdaaebc7332d1ba3b6_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d42a866fb28f41cdaaebc7332d1ba3b6_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:37,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/00cceff5669047fc8b1598b138d0132d, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:37,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/00cceff5669047fc8b1598b138d0132d is 175, key is test_row_0/A:col10/1733645676457/Put/seqid=0 2024-12-08T08:14:37,877 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/51e0e25cc4774806a4e4a082d6d9cea2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/51e0e25cc4774806a4e4a082d6d9cea2 2024-12-08T08:14:37,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741961_1137 (size=31058) 2024-12-08T08:14:37,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741962_1138 (size=30955) 2024-12-08T08:14:37,885 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/00cceff5669047fc8b1598b138d0132d 2024-12-08T08:14:37,885 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/C of 45f279431c7ebbce8a61010c6b97623a into 51e0e25cc4774806a4e4a082d6d9cea2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:37,885 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:37,885 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/C, priority=13, startTime=1733645677335; duration=0sec 2024-12-08T08:14:37,885 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:37,885 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:C 2024-12-08T08:14:37,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/b57fc11f2b174c2680391ffb5367aed7 is 50, key is test_row_0/B:col10/1733645676457/Put/seqid=0 2024-12-08T08:14:37,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741963_1139 (size=12001) 2024-12-08T08:14:37,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645737950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645737950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645737950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645737951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:38,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645738261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645738261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:38,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645738261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645738261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:38,286 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/a0150ee9556a4ee89b50fe7c1f4abe32 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/a0150ee9556a4ee89b50fe7c1f4abe32 2024-12-08T08:14:38,292 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/A of 45f279431c7ebbce8a61010c6b97623a into a0150ee9556a4ee89b50fe7c1f4abe32(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:38,292 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:38,292 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/A, priority=13, startTime=1733645677333; duration=0sec 2024-12-08T08:14:38,292 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:38,292 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:A 2024-12-08T08:14:38,301 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/b57fc11f2b174c2680391ffb5367aed7 2024-12-08T08:14:38,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/c0eaea04c5314f8ba8e355f4d9a8aa94 is 50, key is test_row_0/C:col10/1733645676457/Put/seqid=0 2024-12-08T08:14:38,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741964_1140 (size=12001) 2024-12-08T08:14:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T08:14:38,715 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/c0eaea04c5314f8ba8e355f4d9a8aa94 2024-12-08T08:14:38,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/00cceff5669047fc8b1598b138d0132d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/00cceff5669047fc8b1598b138d0132d 2024-12-08T08:14:38,725 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/00cceff5669047fc8b1598b138d0132d, entries=150, sequenceid=95, filesize=30.2 K 2024-12-08T08:14:38,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/b57fc11f2b174c2680391ffb5367aed7 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/b57fc11f2b174c2680391ffb5367aed7 2024-12-08T08:14:38,730 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/b57fc11f2b174c2680391ffb5367aed7, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T08:14:38,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/c0eaea04c5314f8ba8e355f4d9a8aa94 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/c0eaea04c5314f8ba8e355f4d9a8aa94 2024-12-08T08:14:38,736 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/c0eaea04c5314f8ba8e355f4d9a8aa94, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T08:14:38,737 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 45f279431c7ebbce8a61010c6b97623a in 1356ms, sequenceid=95, compaction requested=false 2024-12-08T08:14:38,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:38,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:38,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-08T08:14:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-08T08:14:38,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-08T08:14:38,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2940 sec 2024-12-08T08:14:38,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.3000 sec 2024-12-08T08:14:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:38,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T08:14:38,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:38,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:38,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:38,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:38,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:38,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:38,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120853a94eda9fa2464a8598370b05b854a4_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645678772/Put/seqid=0 2024-12-08T08:14:38,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645738794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:38,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645738795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:38,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645738797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645738797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:38,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741965_1141 (size=17034) 2024-12-08T08:14:38,803 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:38,809 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120853a94eda9fa2464a8598370b05b854a4_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120853a94eda9fa2464a8598370b05b854a4_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:38,810 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/8ccece8491ce40f28789875cabe021fd, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:38,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/8ccece8491ce40f28789875cabe021fd is 175, key is test_row_0/A:col10/1733645678772/Put/seqid=0 2024-12-08T08:14:38,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741966_1142 (size=48139) 2024-12-08T08:14:38,831 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/8ccece8491ce40f28789875cabe021fd 2024-12-08T08:14:38,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/8b0d8b8f4e92469c9cf771ca270a350b is 50, key is test_row_0/B:col10/1733645678772/Put/seqid=0 2024-12-08T08:14:38,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741967_1143 (size=12001) 2024-12-08T08:14:38,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645738900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:38,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645738900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:38,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:38,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645738910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645738910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:39,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645739110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:39,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645739110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:39,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645739120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:39,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645739121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:39,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/8b0d8b8f4e92469c9cf771ca270a350b 2024-12-08T08:14:39,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/e087830bf8c94cc287e85052414b23f9 is 50, key is test_row_0/C:col10/1733645678772/Put/seqid=0 2024-12-08T08:14:39,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741968_1144 (size=12001) 2024-12-08T08:14:39,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645739420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:39,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645739420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:39,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645739430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:39,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:39,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645739430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:39,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:39,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:39,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42852 deadline: 1733645739660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:39,661 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 45f279431c7ebbce8a61010c6b97623a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:39,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/e087830bf8c94cc287e85052414b23f9 2024-12-08T08:14:39,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/8ccece8491ce40f28789875cabe021fd as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8ccece8491ce40f28789875cabe021fd 2024-12-08T08:14:39,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8ccece8491ce40f28789875cabe021fd, entries=250, sequenceid=112, filesize=47.0 K 2024-12-08T08:14:39,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/8b0d8b8f4e92469c9cf771ca270a350b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/8b0d8b8f4e92469c9cf771ca270a350b 2024-12-08T08:14:39,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/8b0d8b8f4e92469c9cf771ca270a350b, entries=150, sequenceid=112, filesize=11.7 K 2024-12-08T08:14:39,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/e087830bf8c94cc287e85052414b23f9 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/e087830bf8c94cc287e85052414b23f9 2024-12-08T08:14:39,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/e087830bf8c94cc287e85052414b23f9, entries=150, sequenceid=112, filesize=11.7 K 2024-12-08T08:14:39,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 45f279431c7ebbce8a61010c6b97623a in 946ms, sequenceid=112, compaction requested=true 2024-12-08T08:14:39,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:39,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:39,719 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:39,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:39,719 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:39,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:39,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:39,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:39,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:39,721 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:39,721 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/B is initiating minor compaction (all files) 2024-12-08T08:14:39,722 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/B in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:39,722 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ba9a756a6b2b4fbab26a373c8df95ad2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/b57fc11f2b174c2680391ffb5367aed7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/8b0d8b8f4e92469c9cf771ca270a350b] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=35.3 K 2024-12-08T08:14:39,722 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:39,722 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/A is initiating minor compaction (all files) 2024-12-08T08:14:39,722 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/A in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:39,722 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/a0150ee9556a4ee89b50fe7c1f4abe32, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/00cceff5669047fc8b1598b138d0132d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8ccece8491ce40f28789875cabe021fd] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=107.6 K 2024-12-08T08:14:39,722 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:39,722 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/a0150ee9556a4ee89b50fe7c1f4abe32, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/00cceff5669047fc8b1598b138d0132d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8ccece8491ce40f28789875cabe021fd] 2024-12-08T08:14:39,723 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting ba9a756a6b2b4fbab26a373c8df95ad2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733645675734 2024-12-08T08:14:39,723 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0150ee9556a4ee89b50fe7c1f4abe32, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733645675734 2024-12-08T08:14:39,723 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting b57fc11f2b174c2680391ffb5367aed7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733645676450 2024-12-08T08:14:39,723 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00cceff5669047fc8b1598b138d0132d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733645676450 2024-12-08T08:14:39,724 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b0d8b8f4e92469c9cf771ca270a350b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733645677619 2024-12-08T08:14:39,724 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ccece8491ce40f28789875cabe021fd, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733645677619 2024-12-08T08:14:39,739 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:39,750 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#B#compaction#124 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:39,751 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/6d4e239c44374b7fb991a95e50f836fc is 50, key is test_row_0/B:col10/1733645678772/Put/seqid=0 2024-12-08T08:14:39,753 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208b39e91a6ae3a46c997d0c8ba660b4e11_45f279431c7ebbce8a61010c6b97623a store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:39,756 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208b39e91a6ae3a46c997d0c8ba660b4e11_45f279431c7ebbce8a61010c6b97623a, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:39,756 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b39e91a6ae3a46c997d0c8ba660b4e11_45f279431c7ebbce8a61010c6b97623a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:39,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741969_1145 (size=12207) 2024-12-08T08:14:39,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741970_1146 (size=4469) 2024-12-08T08:14:39,772 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#A#compaction#123 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:39,773 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/b201e8f7a9e4473da7a3c3466158291f is 175, key is test_row_0/A:col10/1733645678772/Put/seqid=0 2024-12-08T08:14:39,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741971_1147 (size=31161) 2024-12-08T08:14:39,788 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/b201e8f7a9e4473da7a3c3466158291f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/b201e8f7a9e4473da7a3c3466158291f 2024-12-08T08:14:39,796 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/A of 45f279431c7ebbce8a61010c6b97623a into b201e8f7a9e4473da7a3c3466158291f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:39,796 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:39,796 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/A, priority=13, startTime=1733645679719; duration=0sec 2024-12-08T08:14:39,796 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:39,796 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:A 2024-12-08T08:14:39,796 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:39,798 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:39,798 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/C is initiating minor compaction (all files) 2024-12-08T08:14:39,798 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/C in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:39,799 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/51e0e25cc4774806a4e4a082d6d9cea2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/c0eaea04c5314f8ba8e355f4d9a8aa94, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/e087830bf8c94cc287e85052414b23f9] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=35.3 K 2024-12-08T08:14:39,799 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51e0e25cc4774806a4e4a082d6d9cea2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733645675734 2024-12-08T08:14:39,800 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0eaea04c5314f8ba8e355f4d9a8aa94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733645676450 2024-12-08T08:14:39,800 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e087830bf8c94cc287e85052414b23f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733645677619 2024-12-08T08:14:39,808 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#C#compaction#125 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:39,809 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/45a4fc15c8854d0882f187351dd3d38f is 50, key is test_row_0/C:col10/1733645678772/Put/seqid=0 2024-12-08T08:14:39,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741972_1148 (size=12207) 2024-12-08T08:14:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:39,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T08:14:39,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:39,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:39,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:39,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:39,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:39,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:39,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c010e2e1a2554ca68e84277bce3c6071_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645679931/Put/seqid=0 2024-12-08T08:14:39,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741973_1149 (size=14744) 2024-12-08T08:14:39,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645739950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:39,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:39,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645739979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645739979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645739979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:39,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:39,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645740090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645740090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645740091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645740091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,177 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/6d4e239c44374b7fb991a95e50f836fc as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/6d4e239c44374b7fb991a95e50f836fc 2024-12-08T08:14:40,187 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/B of 45f279431c7ebbce8a61010c6b97623a into 6d4e239c44374b7fb991a95e50f836fc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:40,187 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:40,187 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/B, priority=13, startTime=1733645679719; duration=0sec 2024-12-08T08:14:40,188 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:40,188 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:B 2024-12-08T08:14:40,243 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/45a4fc15c8854d0882f187351dd3d38f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/45a4fc15c8854d0882f187351dd3d38f 2024-12-08T08:14:40,251 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/C of 45f279431c7ebbce8a61010c6b97623a into 45a4fc15c8854d0882f187351dd3d38f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:40,252 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:40,252 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/C, priority=13, startTime=1733645679719; duration=0sec 2024-12-08T08:14:40,252 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:40,252 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:C 2024-12-08T08:14:40,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645740302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645740302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645740302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645740304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,364 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:40,371 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c010e2e1a2554ca68e84277bce3c6071_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c010e2e1a2554ca68e84277bce3c6071_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:40,373 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/1a88df9f404449b597cbf06f1a8ab3f5, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:40,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/1a88df9f404449b597cbf06f1a8ab3f5 is 175, key is test_row_0/A:col10/1733645679931/Put/seqid=0 2024-12-08T08:14:40,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741974_1150 (size=39699) 2024-12-08T08:14:40,380 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/1a88df9f404449b597cbf06f1a8ab3f5 2024-12-08T08:14:40,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/d81079f17e8d4e1a831b50490c49ed0c is 50, key is test_row_0/B:col10/1733645679931/Put/seqid=0 2024-12-08T08:14:40,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741975_1151 (size=12101) 2024-12-08T08:14:40,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/d81079f17e8d4e1a831b50490c49ed0c 2024-12-08T08:14:40,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/061ff3a311be417195326c051d79b92b is 50, key is test_row_0/C:col10/1733645679931/Put/seqid=0 2024-12-08T08:14:40,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741976_1152 (size=12101) 2024-12-08T08:14:40,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/061ff3a311be417195326c051d79b92b 2024-12-08T08:14:40,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/1a88df9f404449b597cbf06f1a8ab3f5 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1a88df9f404449b597cbf06f1a8ab3f5 2024-12-08T08:14:40,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1a88df9f404449b597cbf06f1a8ab3f5, entries=200, sequenceid=133, filesize=38.8 K 2024-12-08T08:14:40,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/d81079f17e8d4e1a831b50490c49ed0c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/d81079f17e8d4e1a831b50490c49ed0c 2024-12-08T08:14:40,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/d81079f17e8d4e1a831b50490c49ed0c, entries=150, sequenceid=133, filesize=11.8 K 2024-12-08T08:14:40,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/061ff3a311be417195326c051d79b92b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/061ff3a311be417195326c051d79b92b 2024-12-08T08:14:40,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/061ff3a311be417195326c051d79b92b, entries=150, sequenceid=133, filesize=11.8 K 2024-12-08T08:14:40,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 45f279431c7ebbce8a61010c6b97623a in 637ms, sequenceid=133, compaction requested=false 2024-12-08T08:14:40,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T08:14:40,601 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:40,602 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-08T08:14:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T08:14:40,605 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:40,606 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:40,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:40,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T08:14:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:40,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:40,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:40,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:40,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:40,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:40,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:40,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ef6caca2eba1482187eb68b98d89dc2f_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645680613/Put/seqid=0 2024-12-08T08:14:40,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645740643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645740644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645740649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645740649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741977_1153 (size=17284) 2024-12-08T08:14:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T08:14:40,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T08:14:40,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:40,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:40,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:40,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:40,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:40,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:40,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645740760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645740760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645740760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645740760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T08:14:40,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:40,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:40,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:40,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:40,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T08:14:40,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645740970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645740971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645740971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645740972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:41,064 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:41,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T08:14:41,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:41,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,069 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ef6caca2eba1482187eb68b98d89dc2f_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ef6caca2eba1482187eb68b98d89dc2f_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:41,070 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/04142441a66c4ac0959b7bee82980aac, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:41,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/04142441a66c4ac0959b7bee82980aac is 175, key is test_row_0/A:col10/1733645680613/Put/seqid=0 2024-12-08T08:14:41,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741978_1154 (size=48389) 2024-12-08T08:14:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T08:14:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T08:14:41,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645741281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:41,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:41,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645741281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:41,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:41,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645741281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:41,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:41,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:41,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645741281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:41,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T08:14:41,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,476 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/04142441a66c4ac0959b7bee82980aac 2024-12-08T08:14:41,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/4451d95452954d78a8d7c034eecf1ad8 is 50, key is test_row_0/B:col10/1733645680613/Put/seqid=0 2024-12-08T08:14:41,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741979_1155 (size=12151) 2024-12-08T08:14:41,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T08:14:41,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:41,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,596 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T08:14:41,597 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-08T08:14:41,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T08:14:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T08:14:41,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645741790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:41,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645741790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:41,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645741801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:41,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645741801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:41,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T08:14:41,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:41,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/4451d95452954d78a8d7c034eecf1ad8 2024-12-08T08:14:41,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/5ba03976e0de47629553d44625bb9f91 is 50, key is test_row_0/C:col10/1733645680613/Put/seqid=0 2024-12-08T08:14:41,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741980_1156 (size=12151) 2024-12-08T08:14:41,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T08:14:41,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:41,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:41,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:42,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T08:14:42,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:42,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:42,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:42,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:42,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:42,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T08:14:42,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:42,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:42,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:42,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:42,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:42,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:42,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/5ba03976e0de47629553d44625bb9f91 2024-12-08T08:14:42,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/04142441a66c4ac0959b7bee82980aac as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/04142441a66c4ac0959b7bee82980aac 2024-12-08T08:14:42,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/04142441a66c4ac0959b7bee82980aac, entries=250, sequenceid=150, filesize=47.3 K 2024-12-08T08:14:42,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/4451d95452954d78a8d7c034eecf1ad8 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/4451d95452954d78a8d7c034eecf1ad8 2024-12-08T08:14:42,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/4451d95452954d78a8d7c034eecf1ad8, entries=150, sequenceid=150, filesize=11.9 K 2024-12-08T08:14:42,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/5ba03976e0de47629553d44625bb9f91 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/5ba03976e0de47629553d44625bb9f91 2024-12-08T08:14:42,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/5ba03976e0de47629553d44625bb9f91, entries=150, sequenceid=150, filesize=11.9 K 2024-12-08T08:14:42,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 45f279431c7ebbce8a61010c6b97623a in 1777ms, sequenceid=150, compaction requested=true 2024-12-08T08:14:42,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:42,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:42,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:42,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:42,392 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:42,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:42,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:42,392 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:42,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:42,393 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:42,393 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:42,393 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/B is initiating minor compaction (all files) 2024-12-08T08:14:42,393 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/A is initiating minor compaction (all files) 2024-12-08T08:14:42,393 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/B in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:42,393 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/A in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:42,393 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/6d4e239c44374b7fb991a95e50f836fc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/d81079f17e8d4e1a831b50490c49ed0c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/4451d95452954d78a8d7c034eecf1ad8] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=35.6 K 2024-12-08T08:14:42,393 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/b201e8f7a9e4473da7a3c3466158291f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1a88df9f404449b597cbf06f1a8ab3f5, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/04142441a66c4ac0959b7bee82980aac] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=116.5 K 2024-12-08T08:14:42,393 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:42,393 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/b201e8f7a9e4473da7a3c3466158291f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1a88df9f404449b597cbf06f1a8ab3f5, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/04142441a66c4ac0959b7bee82980aac] 2024-12-08T08:14:42,394 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d4e239c44374b7fb991a95e50f836fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733645677619 2024-12-08T08:14:42,394 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting b201e8f7a9e4473da7a3c3466158291f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733645677619 2024-12-08T08:14:42,394 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting d81079f17e8d4e1a831b50490c49ed0c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733645678792 2024-12-08T08:14:42,394 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a88df9f404449b597cbf06f1a8ab3f5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733645678792 2024-12-08T08:14:42,394 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 4451d95452954d78a8d7c034eecf1ad8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733645679946 2024-12-08T08:14:42,394 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04142441a66c4ac0959b7bee82980aac, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733645679946 2024-12-08T08:14:42,405 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:42,415 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#B#compaction#133 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:42,415 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/e110f3bea6304628975469c8d99706ef is 50, key is test_row_0/B:col10/1733645680613/Put/seqid=0 2024-12-08T08:14:42,421 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208dc531401c990458c820c64b999c5f50f_45f279431c7ebbce8a61010c6b97623a store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:42,424 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208dc531401c990458c820c64b999c5f50f_45f279431c7ebbce8a61010c6b97623a, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:42,424 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dc531401c990458c820c64b999c5f50f_45f279431c7ebbce8a61010c6b97623a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:42,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741981_1157 (size=12459) 2024-12-08T08:14:42,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741982_1158 (size=4469) 2024-12-08T08:14:42,444 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/e110f3bea6304628975469c8d99706ef as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e110f3bea6304628975469c8d99706ef 2024-12-08T08:14:42,449 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/B of 45f279431c7ebbce8a61010c6b97623a into e110f3bea6304628975469c8d99706ef(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:42,449 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:42,450 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/B, priority=13, startTime=1733645682392; duration=0sec 2024-12-08T08:14:42,450 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:42,450 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:B 2024-12-08T08:14:42,450 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:42,451 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:42,451 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/C is initiating minor compaction (all files) 2024-12-08T08:14:42,451 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/C in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:42,451 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/45a4fc15c8854d0882f187351dd3d38f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/061ff3a311be417195326c051d79b92b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/5ba03976e0de47629553d44625bb9f91] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=35.6 K 2024-12-08T08:14:42,452 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 45a4fc15c8854d0882f187351dd3d38f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733645677619 2024-12-08T08:14:42,452 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 061ff3a311be417195326c051d79b92b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733645678792 2024-12-08T08:14:42,452 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ba03976e0de47629553d44625bb9f91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733645679946 2024-12-08T08:14:42,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T08:14:42,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:42,456 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T08:14:42,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:42,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:42,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:42,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:42,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:42,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:42,460 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#C#compaction#134 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:42,460 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/19c6450281fc42e0b1f25a7e91dfb80e is 50, key is test_row_0/C:col10/1733645680613/Put/seqid=0 2024-12-08T08:14:42,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f0d35d005f46478b9b9a68c263bf5321_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645680646/Put/seqid=0 2024-12-08T08:14:42,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741983_1159 (size=12459) 2024-12-08T08:14:42,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741984_1160 (size=12304) 2024-12-08T08:14:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T08:14:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:42,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:42,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645742818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:42,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645742821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:42,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645742821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:42,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645742825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:42,825 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 45f279431c7ebbce8a61010c6b97623a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:42,826 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 45f279431c7ebbce8a61010c6b97623a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:42,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:42,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:42,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:42,839 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#A#compaction#132 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:42,840 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/e08e4e79e7b24b07bfc461aed4ccbf50 is 175, key is test_row_0/A:col10/1733645680613/Put/seqid=0 2024-12-08T08:14:42,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741985_1161 (size=31413) 2024-12-08T08:14:42,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:42,872 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/19c6450281fc42e0b1f25a7e91dfb80e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/19c6450281fc42e0b1f25a7e91dfb80e 2024-12-08T08:14:42,874 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f0d35d005f46478b9b9a68c263bf5321_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f0d35d005f46478b9b9a68c263bf5321_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:42,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/5d82fa2430d447d88b36fe5f0dcf5ab1, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:42,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/5d82fa2430d447d88b36fe5f0dcf5ab1 is 175, key is test_row_0/A:col10/1733645680646/Put/seqid=0 2024-12-08T08:14:42,877 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/C of 45f279431c7ebbce8a61010c6b97623a into 19c6450281fc42e0b1f25a7e91dfb80e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:42,877 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:42,877 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/C, priority=13, startTime=1733645682392; duration=0sec 2024-12-08T08:14:42,877 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:42,877 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:C 2024-12-08T08:14:42,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741986_1162 (size=31105) 2024-12-08T08:14:42,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645742931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:42,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645742931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:43,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645743140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:43,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645743140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:43,249 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/e08e4e79e7b24b07bfc461aed4ccbf50 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e08e4e79e7b24b07bfc461aed4ccbf50 2024-12-08T08:14:43,254 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/A of 45f279431c7ebbce8a61010c6b97623a into e08e4e79e7b24b07bfc461aed4ccbf50(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:43,254 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:43,254 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/A, priority=13, startTime=1733645682392; duration=0sec 2024-12-08T08:14:43,254 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:43,255 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:A 2024-12-08T08:14:43,280 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/5d82fa2430d447d88b36fe5f0dcf5ab1 2024-12-08T08:14:43,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/3bc63148d5814449a544fee774ccb5a9 is 50, key is test_row_0/B:col10/1733645680646/Put/seqid=0 2024-12-08T08:14:43,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741987_1163 (size=12151) 2024-12-08T08:14:43,291 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/3bc63148d5814449a544fee774ccb5a9 2024-12-08T08:14:43,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/cdefbc56041340999764f22082a3337d is 50, key is test_row_0/C:col10/1733645680646/Put/seqid=0 2024-12-08T08:14:43,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741988_1164 (size=12151) 2024-12-08T08:14:43,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645743451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:43,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645743451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:43,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42852 deadline: 1733645743700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:43,705 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/cdefbc56041340999764f22082a3337d 2024-12-08T08:14:43,705 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 45f279431c7ebbce8a61010c6b97623a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/5d82fa2430d447d88b36fe5f0dcf5ab1 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/5d82fa2430d447d88b36fe5f0dcf5ab1 2024-12-08T08:14:43,716 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/5d82fa2430d447d88b36fe5f0dcf5ab1, entries=150, sequenceid=169, filesize=30.4 K 2024-12-08T08:14:43,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/3bc63148d5814449a544fee774ccb5a9 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3bc63148d5814449a544fee774ccb5a9 2024-12-08T08:14:43,722 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3bc63148d5814449a544fee774ccb5a9, entries=150, sequenceid=169, filesize=11.9 K 2024-12-08T08:14:43,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/cdefbc56041340999764f22082a3337d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/cdefbc56041340999764f22082a3337d 2024-12-08T08:14:43,729 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/cdefbc56041340999764f22082a3337d, entries=150, sequenceid=169, filesize=11.9 K 2024-12-08T08:14:43,730 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 45f279431c7ebbce8a61010c6b97623a in 1274ms, sequenceid=169, compaction requested=false 2024-12-08T08:14:43,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:43,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:43,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-08T08:14:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-08T08:14:43,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-08T08:14:43,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1240 sec 2024-12-08T08:14:43,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 3.1310 sec 2024-12-08T08:14:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:43,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T08:14:43,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:43,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:43,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:43,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:43,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:43,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:43,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120809b1537397ad4f08a084ca2f06609a04_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645683961/Put/seqid=0 2024-12-08T08:14:43,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741989_1165 (size=17284) 2024-12-08T08:14:43,997 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:44,001 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120809b1537397ad4f08a084ca2f06609a04_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120809b1537397ad4f08a084ca2f06609a04_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:44,002 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/fd2964037d9f4693ab1ce023ef77f130, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:44,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/fd2964037d9f4693ab1ce023ef77f130 is 175, key is test_row_0/A:col10/1733645683961/Put/seqid=0 2024-12-08T08:14:44,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741990_1166 (size=48389) 2024-12-08T08:14:44,023 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/fd2964037d9f4693ab1ce023ef77f130 2024-12-08T08:14:44,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645744027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:44,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/94e5ae3d5f8b4a90b72a30dac6d8f334 is 50, key is test_row_0/B:col10/1733645683961/Put/seqid=0 2024-12-08T08:14:44,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645744035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:44,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741991_1167 (size=12151) 2024-12-08T08:14:44,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/94e5ae3d5f8b4a90b72a30dac6d8f334 2024-12-08T08:14:44,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/ee045b44503e44b1b89e41d8f3c2cf27 is 50, key is test_row_0/C:col10/1733645683961/Put/seqid=0 2024-12-08T08:14:44,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741992_1168 (size=12151) 2024-12-08T08:14:44,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/ee045b44503e44b1b89e41d8f3c2cf27 2024-12-08T08:14:44,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/fd2964037d9f4693ab1ce023ef77f130 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/fd2964037d9f4693ab1ce023ef77f130 2024-12-08T08:14:44,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/fd2964037d9f4693ab1ce023ef77f130, entries=250, sequenceid=187, filesize=47.3 K 2024-12-08T08:14:44,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/94e5ae3d5f8b4a90b72a30dac6d8f334 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/94e5ae3d5f8b4a90b72a30dac6d8f334 2024-12-08T08:14:44,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/94e5ae3d5f8b4a90b72a30dac6d8f334, entries=150, sequenceid=187, filesize=11.9 K 2024-12-08T08:14:44,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/ee045b44503e44b1b89e41d8f3c2cf27 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/ee045b44503e44b1b89e41d8f3c2cf27 2024-12-08T08:14:44,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/ee045b44503e44b1b89e41d8f3c2cf27, entries=150, sequenceid=187, filesize=11.9 K 2024-12-08T08:14:44,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 45f279431c7ebbce8a61010c6b97623a in 168ms, sequenceid=187, compaction requested=true 2024-12-08T08:14:44,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:44,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:44,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:44,132 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:44,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:44,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:44,132 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:44,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:44,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:44,134 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:44,134 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/B is initiating minor compaction (all files) 2024-12-08T08:14:44,134 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/B in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:44,134 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e110f3bea6304628975469c8d99706ef, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3bc63148d5814449a544fee774ccb5a9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/94e5ae3d5f8b4a90b72a30dac6d8f334] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=35.9 K 2024-12-08T08:14:44,135 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:44,135 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/A is initiating minor compaction (all files) 2024-12-08T08:14:44,135 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/A in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:44,135 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e08e4e79e7b24b07bfc461aed4ccbf50, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/5d82fa2430d447d88b36fe5f0dcf5ab1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/fd2964037d9f4693ab1ce023ef77f130] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=108.3 K 2024-12-08T08:14:44,135 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:44,135 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e08e4e79e7b24b07bfc461aed4ccbf50, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/5d82fa2430d447d88b36fe5f0dcf5ab1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/fd2964037d9f4693ab1ce023ef77f130] 2024-12-08T08:14:44,136 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting e110f3bea6304628975469c8d99706ef, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733645679946 2024-12-08T08:14:44,137 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e08e4e79e7b24b07bfc461aed4ccbf50, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733645679946 2024-12-08T08:14:44,137 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bc63148d5814449a544fee774ccb5a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733645680642 2024-12-08T08:14:44,138 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d82fa2430d447d88b36fe5f0dcf5ab1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733645680642 2024-12-08T08:14:44,138 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 94e5ae3d5f8b4a90b72a30dac6d8f334, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733645682817 2024-12-08T08:14:44,138 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd2964037d9f4693ab1ce023ef77f130, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733645682814 2024-12-08T08:14:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:44,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T08:14:44,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:44,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:44,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:44,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:44,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:44,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:44,157 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:44,157 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/871bc02bf7f64c999efaee90e66228dc is 50, key is test_row_0/B:col10/1733645683961/Put/seqid=0 2024-12-08T08:14:44,157 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:44,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208395eb77371374f9eafb74bb77e841963_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645684141/Put/seqid=0 2024-12-08T08:14:44,161 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120852cc00422700470eb3a3aedde6349478_45f279431c7ebbce8a61010c6b97623a store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:44,163 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120852cc00422700470eb3a3aedde6349478_45f279431c7ebbce8a61010c6b97623a, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:44,163 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120852cc00422700470eb3a3aedde6349478_45f279431c7ebbce8a61010c6b97623a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:44,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741993_1169 (size=12561) 2024-12-08T08:14:44,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645744176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:44,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:44,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741994_1170 (size=12304) 2024-12-08T08:14:44,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:44,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741995_1171 (size=4469) 2024-12-08T08:14:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645744180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:44,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:44,185 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#A#compaction#142 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:44,185 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/60e883e7d0f8439981151094ef7e68f5 is 175, key is test_row_0/A:col10/1733645683961/Put/seqid=0 2024-12-08T08:14:44,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741996_1172 (size=31515) 2024-12-08T08:14:44,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645744290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645744290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:44,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645744501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:44,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:44,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645744501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:44,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:44,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:44,584 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/871bc02bf7f64c999efaee90e66228dc as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/871bc02bf7f64c999efaee90e66228dc 2024-12-08T08:14:44,584 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:44,590 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208395eb77371374f9eafb74bb77e841963_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208395eb77371374f9eafb74bb77e841963_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:44,592 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/8701d62816f24fc4945116c5fb5ba31c, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:44,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/8701d62816f24fc4945116c5fb5ba31c is 175, key is test_row_0/A:col10/1733645684141/Put/seqid=0 2024-12-08T08:14:44,593 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/B of 45f279431c7ebbce8a61010c6b97623a into 871bc02bf7f64c999efaee90e66228dc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:44,593 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:44,593 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/B, priority=13, startTime=1733645684132; duration=0sec 2024-12-08T08:14:44,593 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:44,593 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:B 2024-12-08T08:14:44,593 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:44,597 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:44,597 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/C is initiating minor compaction (all files) 2024-12-08T08:14:44,597 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/C in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:44,597 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/19c6450281fc42e0b1f25a7e91dfb80e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/cdefbc56041340999764f22082a3337d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/ee045b44503e44b1b89e41d8f3c2cf27] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=35.9 K 2024-12-08T08:14:44,598 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/60e883e7d0f8439981151094ef7e68f5 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/60e883e7d0f8439981151094ef7e68f5 2024-12-08T08:14:44,598 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c6450281fc42e0b1f25a7e91dfb80e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733645679946 2024-12-08T08:14:44,599 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting cdefbc56041340999764f22082a3337d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733645680642 2024-12-08T08:14:44,600 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting ee045b44503e44b1b89e41d8f3c2cf27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733645682817 2024-12-08T08:14:44,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741997_1173 (size=31105) 2024-12-08T08:14:44,605 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/8701d62816f24fc4945116c5fb5ba31c 2024-12-08T08:14:44,605 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/A of 45f279431c7ebbce8a61010c6b97623a into 60e883e7d0f8439981151094ef7e68f5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:44,605 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:44,605 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/A, priority=13, startTime=1733645684132; duration=0sec 2024-12-08T08:14:44,605 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:44,605 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:A 2024-12-08T08:14:44,616 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#C#compaction#145 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:44,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/50aace9e1d1e442587b198646c316c07 is 50, key is test_row_0/B:col10/1733645684141/Put/seqid=0 2024-12-08T08:14:44,617 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/4eab6745862a485e92111bf74644c910 is 50, key is test_row_0/C:col10/1733645683961/Put/seqid=0 2024-12-08T08:14:44,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741999_1175 (size=12561) 2024-12-08T08:14:44,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741998_1174 (size=12151) 2024-12-08T08:14:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T08:14:44,761 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:44,762 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-08T08:14:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T08:14:44,763 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:44,764 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:44,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:44,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645744810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:44,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:44,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645744810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:44,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:44,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645744841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:44,841 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 45f279431c7ebbce8a61010c6b97623a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:44,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645744850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:44,851 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 45f279431c7ebbce8a61010c6b97623a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:44,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T08:14:44,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T08:14:44,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:44,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:44,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:44,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:44,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:44,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:45,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/50aace9e1d1e442587b198646c316c07 2024-12-08T08:14:45,040 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/4eab6745862a485e92111bf74644c910 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/4eab6745862a485e92111bf74644c910 2024-12-08T08:14:45,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/cfe442c60b6e4d6db3f1eb815e9d6188 is 50, key is test_row_0/C:col10/1733645684141/Put/seqid=0 2024-12-08T08:14:45,047 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/C of 45f279431c7ebbce8a61010c6b97623a into 4eab6745862a485e92111bf74644c910(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:45,047 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:45,047 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/C, priority=13, startTime=1733645684132; duration=0sec 2024-12-08T08:14:45,048 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:45,048 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:C 2024-12-08T08:14:45,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742000_1176 (size=12151) 2024-12-08T08:14:45,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/cfe442c60b6e4d6db3f1eb815e9d6188 2024-12-08T08:14:45,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/8701d62816f24fc4945116c5fb5ba31c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8701d62816f24fc4945116c5fb5ba31c 2024-12-08T08:14:45,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8701d62816f24fc4945116c5fb5ba31c, entries=150, sequenceid=205, filesize=30.4 K 2024-12-08T08:14:45,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/50aace9e1d1e442587b198646c316c07 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/50aace9e1d1e442587b198646c316c07 2024-12-08T08:14:45,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/50aace9e1d1e442587b198646c316c07, entries=150, sequenceid=205, filesize=11.9 K 2024-12-08T08:14:45,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/cfe442c60b6e4d6db3f1eb815e9d6188 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/cfe442c60b6e4d6db3f1eb815e9d6188 2024-12-08T08:14:45,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T08:14:45,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:45,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:45,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:45,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:45,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:45,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/cfe442c60b6e4d6db3f1eb815e9d6188, entries=150, sequenceid=205, filesize=11.9 K 2024-12-08T08:14:45,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 45f279431c7ebbce8a61010c6b97623a in 930ms, sequenceid=205, compaction requested=false 2024-12-08T08:14:45,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T08:14:45,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T08:14:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:45,222 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T08:14:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:45,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:45,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:45,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085b254c4742d5421ba25eaead16fb250a_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645684153/Put/seqid=0 2024-12-08T08:14:45,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742001_1177 (size=12304) 2024-12-08T08:14:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:45,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:45,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645745352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:45,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645745353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T08:14:45,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645745460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:45,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645745460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:45,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:45,639 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085b254c4742d5421ba25eaead16fb250a_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085b254c4742d5421ba25eaead16fb250a_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:45,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/2b635dd917554df8931d7a3932035acc, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:45,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/2b635dd917554df8931d7a3932035acc is 175, key is test_row_0/A:col10/1733645684153/Put/seqid=0 2024-12-08T08:14:45,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742002_1178 (size=31105) 2024-12-08T08:14:45,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645745670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:45,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645745670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T08:14:45,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645745980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645745981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:46,047 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/2b635dd917554df8931d7a3932035acc 2024-12-08T08:14:46,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/1413920a7bfe4377b5568bd99ecb1f28 is 50, key is test_row_0/B:col10/1733645684153/Put/seqid=0 2024-12-08T08:14:46,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742003_1179 (size=12151) 2024-12-08T08:14:46,058 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/1413920a7bfe4377b5568bd99ecb1f28 2024-12-08T08:14:46,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/8d357626d1a341e796844fe03e53ed45 is 50, key is test_row_0/C:col10/1733645684153/Put/seqid=0 2024-12-08T08:14:46,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742004_1180 (size=12151) 2024-12-08T08:14:46,374 INFO [master/422e41b75614:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-08T08:14:46,374 INFO [master/422e41b75614:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-08T08:14:46,469 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/8d357626d1a341e796844fe03e53ed45 2024-12-08T08:14:46,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/2b635dd917554df8931d7a3932035acc as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/2b635dd917554df8931d7a3932035acc 2024-12-08T08:14:46,478 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/2b635dd917554df8931d7a3932035acc, entries=150, sequenceid=222, filesize=30.4 K 2024-12-08T08:14:46,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/1413920a7bfe4377b5568bd99ecb1f28 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/1413920a7bfe4377b5568bd99ecb1f28 2024-12-08T08:14:46,483 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/1413920a7bfe4377b5568bd99ecb1f28, entries=150, sequenceid=222, filesize=11.9 K 2024-12-08T08:14:46,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/8d357626d1a341e796844fe03e53ed45 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/8d357626d1a341e796844fe03e53ed45 2024-12-08T08:14:46,488 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/8d357626d1a341e796844fe03e53ed45, entries=150, sequenceid=222, filesize=11.9 K 2024-12-08T08:14:46,489 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 45f279431c7ebbce8a61010c6b97623a in 1267ms, sequenceid=222, compaction requested=true 2024-12-08T08:14:46,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:46,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:46,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-08T08:14:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-08T08:14:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:46,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T08:14:46,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:46,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:46,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:46,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:46,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:46,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:46,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-08T08:14:46,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7270 sec 2024-12-08T08:14:46,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.7310 sec 2024-12-08T08:14:46,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208da32fc2b54b049539fa6fc2c4020af86_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645685339/Put/seqid=0 2024-12-08T08:14:46,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742005_1181 (size=14794) 2024-12-08T08:14:46,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645746508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:46,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645746512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:46,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:46,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645746621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645746621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645746830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645746830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:46,909 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T08:14:46,910 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:46,911 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-08T08:14:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T08:14:46,914 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208da32fc2b54b049539fa6fc2c4020af86_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208da32fc2b54b049539fa6fc2c4020af86_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:46,915 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:46,915 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/87664ffafdcb4cbc8e3f6dfb7840cd58, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:46,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/87664ffafdcb4cbc8e3f6dfb7840cd58 is 175, key is test_row_0/A:col10/1733645685339/Put/seqid=0 2024-12-08T08:14:46,916 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:46,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:46,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742006_1182 (size=39749) 2024-12-08T08:14:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T08:14:47,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T08:14:47,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:47,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645747141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:47,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645747141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:47,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T08:14:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:47,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T08:14:47,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T08:14:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,380 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/87664ffafdcb4cbc8e3f6dfb7840cd58 2024-12-08T08:14:47,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/4ecc3c3b039745a08c3fb55f612040f7 is 50, key is test_row_0/B:col10/1733645685339/Put/seqid=0 2024-12-08T08:14:47,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742007_1183 (size=12151) 2024-12-08T08:14:47,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T08:14:47,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:47,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T08:14:47,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645747651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:47,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:47,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:47,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645747651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:47,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:47,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:47,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:47,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:47,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T08:14:47,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:47,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/4ecc3c3b039745a08c3fb55f612040f7 2024-12-08T08:14:47,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/c3ee9b417cbf4837a7aac4008ae5c989 is 50, key is test_row_0/C:col10/1733645685339/Put/seqid=0 2024-12-08T08:14:47,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742008_1184 (size=12151) 2024-12-08T08:14:47,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T08:14:47,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:47,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T08:14:47,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:47,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:47,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T08:14:48,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T08:14:48,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:48,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:48,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:48,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:48,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:48,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/c3ee9b417cbf4837a7aac4008ae5c989 2024-12-08T08:14:48,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/87664ffafdcb4cbc8e3f6dfb7840cd58 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/87664ffafdcb4cbc8e3f6dfb7840cd58 2024-12-08T08:14:48,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/87664ffafdcb4cbc8e3f6dfb7840cd58, entries=200, sequenceid=241, filesize=38.8 K 2024-12-08T08:14:48,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/4ecc3c3b039745a08c3fb55f612040f7 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/4ecc3c3b039745a08c3fb55f612040f7 2024-12-08T08:14:48,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/4ecc3c3b039745a08c3fb55f612040f7, entries=150, sequenceid=241, filesize=11.9 K 2024-12-08T08:14:48,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/c3ee9b417cbf4837a7aac4008ae5c989 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/c3ee9b417cbf4837a7aac4008ae5c989 2024-12-08T08:14:48,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/c3ee9b417cbf4837a7aac4008ae5c989, entries=150, sequenceid=241, filesize=11.9 K 2024-12-08T08:14:48,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 45f279431c7ebbce8a61010c6b97623a in 1733ms, sequenceid=241, compaction requested=true 2024-12-08T08:14:48,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:48,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:48,226 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:48,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:48,226 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:48,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:48,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:48,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:48,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:48,228 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133474 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:48,228 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/A is initiating minor compaction (all files) 2024-12-08T08:14:48,228 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/A in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:48,228 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/60e883e7d0f8439981151094ef7e68f5, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8701d62816f24fc4945116c5fb5ba31c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/2b635dd917554df8931d7a3932035acc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/87664ffafdcb4cbc8e3f6dfb7840cd58] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=130.3 K 2024-12-08T08:14:48,228 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:48,228 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/60e883e7d0f8439981151094ef7e68f5, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8701d62816f24fc4945116c5fb5ba31c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/2b635dd917554df8931d7a3932035acc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/87664ffafdcb4cbc8e3f6dfb7840cd58] 2024-12-08T08:14:48,228 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:48,229 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/B is initiating minor compaction (all files) 2024-12-08T08:14:48,229 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/B in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:48,229 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/871bc02bf7f64c999efaee90e66228dc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/50aace9e1d1e442587b198646c316c07, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/1413920a7bfe4377b5568bd99ecb1f28, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/4ecc3c3b039745a08c3fb55f612040f7] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=47.9 K 2024-12-08T08:14:48,229 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60e883e7d0f8439981151094ef7e68f5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733645682817 2024-12-08T08:14:48,229 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 871bc02bf7f64c999efaee90e66228dc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733645682817 2024-12-08T08:14:48,229 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8701d62816f24fc4945116c5fb5ba31c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733645684024 2024-12-08T08:14:48,230 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 50aace9e1d1e442587b198646c316c07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733645684024 2024-12-08T08:14:48,230 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b635dd917554df8931d7a3932035acc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733645684153 2024-12-08T08:14:48,230 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 1413920a7bfe4377b5568bd99ecb1f28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733645684153 2024-12-08T08:14:48,230 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ecc3c3b039745a08c3fb55f612040f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733645685339 2024-12-08T08:14:48,231 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87664ffafdcb4cbc8e3f6dfb7840cd58, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733645685339 2024-12-08T08:14:48,242 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#B#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:48,243 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/263ccfb4df4942ca8512122be2c26246 is 50, key is test_row_0/B:col10/1733645685339/Put/seqid=0 2024-12-08T08:14:48,247 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:48,253 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208bc2ade2cf9864cf88acbf20847c72e2b_45f279431c7ebbce8a61010c6b97623a store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:48,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742009_1185 (size=12697) 2024-12-08T08:14:48,255 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208bc2ade2cf9864cf88acbf20847c72e2b_45f279431c7ebbce8a61010c6b97623a, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:48,255 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bc2ade2cf9864cf88acbf20847c72e2b_45f279431c7ebbce8a61010c6b97623a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:48,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742010_1186 (size=4469) 2024-12-08T08:14:48,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T08:14:48,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:48,306 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T08:14:48,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:48,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:48,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086e0e3c97dcb44fb8b2cbc5f921e3d32b_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645686510/Put/seqid=0 2024-12-08T08:14:48,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742011_1187 (size=12304) 2024-12-08T08:14:48,659 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/263ccfb4df4942ca8512122be2c26246 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/263ccfb4df4942ca8512122be2c26246 2024-12-08T08:14:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:48,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:48,666 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#A#compaction#154 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:48,667 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/B of 45f279431c7ebbce8a61010c6b97623a into 263ccfb4df4942ca8512122be2c26246(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:48,667 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:48,667 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/B, priority=12, startTime=1733645688226; duration=0sec 2024-12-08T08:14:48,667 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/9ce6e6fb9edb4c22bee18d2ef4b71a4a is 175, key is test_row_0/A:col10/1733645685339/Put/seqid=0 2024-12-08T08:14:48,667 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:48,667 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:B 2024-12-08T08:14:48,667 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:48,669 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:48,669 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/C is initiating minor compaction (all files) 2024-12-08T08:14:48,669 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/C in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:48,669 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/4eab6745862a485e92111bf74644c910, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/cfe442c60b6e4d6db3f1eb815e9d6188, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/8d357626d1a341e796844fe03e53ed45, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/c3ee9b417cbf4837a7aac4008ae5c989] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=47.9 K 2024-12-08T08:14:48,670 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eab6745862a485e92111bf74644c910, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733645682817 2024-12-08T08:14:48,670 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting cfe442c60b6e4d6db3f1eb815e9d6188, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733645684024 2024-12-08T08:14:48,670 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d357626d1a341e796844fe03e53ed45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733645684153 2024-12-08T08:14:48,671 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting c3ee9b417cbf4837a7aac4008ae5c989, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733645685339 2024-12-08T08:14:48,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742012_1188 (size=31651) 2024-12-08T08:14:48,686 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:48,687 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/83889a7ef56949f7bbb395dea554bcdb is 50, key is test_row_0/C:col10/1733645685339/Put/seqid=0 2024-12-08T08:14:48,687 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/9ce6e6fb9edb4c22bee18d2ef4b71a4a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/9ce6e6fb9edb4c22bee18d2ef4b71a4a 2024-12-08T08:14:48,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742013_1189 (size=12697) 2024-12-08T08:14:48,697 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/A of 45f279431c7ebbce8a61010c6b97623a into 9ce6e6fb9edb4c22bee18d2ef4b71a4a(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:48,697 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:48,697 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/A, priority=12, startTime=1733645688226; duration=0sec 2024-12-08T08:14:48,697 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:48,697 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:A 2024-12-08T08:14:48,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645748702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:48,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645748704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:48,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:48,725 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086e0e3c97dcb44fb8b2cbc5f921e3d32b_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086e0e3c97dcb44fb8b2cbc5f921e3d32b_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:48,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/e3b111040d9c46c5adba015a0a6f20a0, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:48,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/e3b111040d9c46c5adba015a0a6f20a0 is 175, key is test_row_0/A:col10/1733645686510/Put/seqid=0 2024-12-08T08:14:48,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742014_1190 (size=31105) 2024-12-08T08:14:48,732 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/e3b111040d9c46c5adba015a0a6f20a0 2024-12-08T08:14:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/f3ec4765b841414a9b6c90a4e24f56b8 is 50, key is test_row_0/B:col10/1733645686510/Put/seqid=0 2024-12-08T08:14:48,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742015_1191 (size=12151) 2024-12-08T08:14:48,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645748810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:48,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:48,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645748810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:48,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42810 deadline: 1733645748861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:48,864 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 45f279431c7ebbce8a61010c6b97623a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:48,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:48,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:48,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:48,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42818 deadline: 1733645748871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:48,874 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 45f279431c7ebbce8a61010c6b97623a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:49,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645749020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645749020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T08:14:49,098 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/83889a7ef56949f7bbb395dea554bcdb as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/83889a7ef56949f7bbb395dea554bcdb 2024-12-08T08:14:49,103 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/C of 45f279431c7ebbce8a61010c6b97623a into 83889a7ef56949f7bbb395dea554bcdb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:49,104 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:49,104 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/C, priority=12, startTime=1733645688226; duration=0sec 2024-12-08T08:14:49,104 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:49,104 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:C 2024-12-08T08:14:49,143 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/f3ec4765b841414a9b6c90a4e24f56b8 2024-12-08T08:14:49,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/9e7e9095fd7d4d3eaff895a06f08dc91 is 50, key is test_row_0/C:col10/1733645686510/Put/seqid=0 2024-12-08T08:14:49,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742016_1192 (size=12151) 2024-12-08T08:14:49,158 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/9e7e9095fd7d4d3eaff895a06f08dc91 2024-12-08T08:14:49,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/e3b111040d9c46c5adba015a0a6f20a0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e3b111040d9c46c5adba015a0a6f20a0 2024-12-08T08:14:49,167 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e3b111040d9c46c5adba015a0a6f20a0, entries=150, sequenceid=254, filesize=30.4 K 2024-12-08T08:14:49,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/f3ec4765b841414a9b6c90a4e24f56b8 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/f3ec4765b841414a9b6c90a4e24f56b8 2024-12-08T08:14:49,174 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/f3ec4765b841414a9b6c90a4e24f56b8, entries=150, sequenceid=254, filesize=11.9 K 2024-12-08T08:14:49,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/9e7e9095fd7d4d3eaff895a06f08dc91 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/9e7e9095fd7d4d3eaff895a06f08dc91 2024-12-08T08:14:49,181 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/9e7e9095fd7d4d3eaff895a06f08dc91, entries=150, sequenceid=254, filesize=11.9 K 2024-12-08T08:14:49,182 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 45f279431c7ebbce8a61010c6b97623a in 876ms, sequenceid=254, compaction requested=false 2024-12-08T08:14:49,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:49,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:49,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-08T08:14:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-08T08:14:49,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-08T08:14:49,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2670 sec 2024-12-08T08:14:49,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.2750 sec 2024-12-08T08:14:49,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T08:14:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:49,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:49,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:49,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:49,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:49,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:49,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:49,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b7ca789b76bf4fba9a6f7807d7af6e75_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645688696/Put/seqid=0 2024-12-08T08:14:49,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742017_1193 (size=17534) 2024-12-08T08:14:49,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645749353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:49,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645749356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:49,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:49,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645749461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:49,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:49,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645749470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:49,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645749670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:49,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:49,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645749681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:49,755 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:49,759 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b7ca789b76bf4fba9a6f7807d7af6e75_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b7ca789b76bf4fba9a6f7807d7af6e75_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:49,760 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/f6207fdd88cf41c6854180de3446799b, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:49,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/f6207fdd88cf41c6854180de3446799b is 175, key is test_row_0/A:col10/1733645688696/Put/seqid=0 2024-12-08T08:14:49,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742018_1194 (size=48639) 2024-12-08T08:14:49,773 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/f6207fdd88cf41c6854180de3446799b 2024-12-08T08:14:49,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/3162fd06373b4b6bb6c4baef98b83f58 is 50, key is test_row_0/B:col10/1733645688696/Put/seqid=0 2024-12-08T08:14:49,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742019_1195 (size=12301) 2024-12-08T08:14:49,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/3162fd06373b4b6bb6c4baef98b83f58 2024-12-08T08:14:49,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/1436c99689164124b8607a746d46fcd4 is 50, key is test_row_0/C:col10/1733645688696/Put/seqid=0 2024-12-08T08:14:49,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742020_1196 (size=12301) 2024-12-08T08:14:49,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645749980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:49,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:49,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645749990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:50,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/1436c99689164124b8607a746d46fcd4 2024-12-08T08:14:50,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/f6207fdd88cf41c6854180de3446799b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/f6207fdd88cf41c6854180de3446799b 2024-12-08T08:14:50,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/f6207fdd88cf41c6854180de3446799b, entries=250, sequenceid=278, filesize=47.5 K 2024-12-08T08:14:50,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/3162fd06373b4b6bb6c4baef98b83f58 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3162fd06373b4b6bb6c4baef98b83f58 2024-12-08T08:14:50,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3162fd06373b4b6bb6c4baef98b83f58, entries=150, sequenceid=278, filesize=12.0 K 2024-12-08T08:14:50,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/1436c99689164124b8607a746d46fcd4 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/1436c99689164124b8607a746d46fcd4 2024-12-08T08:14:50,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/1436c99689164124b8607a746d46fcd4, entries=150, sequenceid=278, filesize=12.0 K 2024-12-08T08:14:50,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 45f279431c7ebbce8a61010c6b97623a in 887ms, sequenceid=278, compaction requested=true 2024-12-08T08:14:50,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:50,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:50,222 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:50,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:50,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:50,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:50,222 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:50,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:50,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:50,223 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111395 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:50,223 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:50,223 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/A is initiating minor compaction (all files) 2024-12-08T08:14:50,223 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/B is initiating minor compaction (all files) 2024-12-08T08:14:50,223 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/B in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:50,223 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/A in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:50,223 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/263ccfb4df4942ca8512122be2c26246, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/f3ec4765b841414a9b6c90a4e24f56b8, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3162fd06373b4b6bb6c4baef98b83f58] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=36.3 K 2024-12-08T08:14:50,223 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/9ce6e6fb9edb4c22bee18d2ef4b71a4a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e3b111040d9c46c5adba015a0a6f20a0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/f6207fdd88cf41c6854180de3446799b] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=108.8 K 2024-12-08T08:14:50,223 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:50,223 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/9ce6e6fb9edb4c22bee18d2ef4b71a4a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e3b111040d9c46c5adba015a0a6f20a0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/f6207fdd88cf41c6854180de3446799b] 2024-12-08T08:14:50,223 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 263ccfb4df4942ca8512122be2c26246, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733645685339 2024-12-08T08:14:50,224 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ce6e6fb9edb4c22bee18d2ef4b71a4a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733645685339 2024-12-08T08:14:50,224 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting f3ec4765b841414a9b6c90a4e24f56b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733645686502 2024-12-08T08:14:50,224 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3b111040d9c46c5adba015a0a6f20a0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733645686502 2024-12-08T08:14:50,224 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 3162fd06373b4b6bb6c4baef98b83f58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733645688696 2024-12-08T08:14:50,224 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6207fdd88cf41c6854180de3446799b, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733645688696 2024-12-08T08:14:50,230 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#B#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:50,231 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/ca657c413d894313a94695c09985f810 is 50, key is test_row_0/B:col10/1733645688696/Put/seqid=0 2024-12-08T08:14:50,233 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:50,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742021_1197 (size=12949) 2024-12-08T08:14:50,235 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412087e03ed7e45564cddab69ca77440d7ce0_45f279431c7ebbce8a61010c6b97623a store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:50,236 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412087e03ed7e45564cddab69ca77440d7ce0_45f279431c7ebbce8a61010c6b97623a, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:50,236 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087e03ed7e45564cddab69ca77440d7ce0_45f279431c7ebbce8a61010c6b97623a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:50,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742022_1198 (size=4469) 2024-12-08T08:14:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:50,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T08:14:50,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:50,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:50,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:50,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:50,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:50,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:50,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084fdac473dd6f4d539e49ecca66b502a2_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645689353/Put/seqid=0 2024-12-08T08:14:50,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742023_1199 (size=14994) 2024-12-08T08:14:50,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733645690522, max timestamp=1733645690535Num uniques 859; 2024-12-08T08:14:50,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733645690522, max timestamp=1733645690535Num uniques 859; 2024-12-08T08:14:50,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:14:50,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:14:50,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733645690522, max timestamp=1733645690535Num uniques 859; 2024-12-08T08:14:50,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:14:50,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645750586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:50,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645750587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:50,639 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/ca657c413d894313a94695c09985f810 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ca657c413d894313a94695c09985f810 2024-12-08T08:14:50,644 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#A#compaction#163 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:50,644 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/3c01c664367d433bb215838be2d7f7e2 is 175, key is test_row_0/A:col10/1733645688696/Put/seqid=0 2024-12-08T08:14:50,648 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/B of 45f279431c7ebbce8a61010c6b97623a into ca657c413d894313a94695c09985f810(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:50,648 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:50,648 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/B, priority=13, startTime=1733645690222; duration=0sec 2024-12-08T08:14:50,648 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:50,648 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:B 2024-12-08T08:14:50,648 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:14:50,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742024_1200 (size=31903) 2024-12-08T08:14:50,650 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:14:50,650 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/C is initiating minor compaction (all files) 2024-12-08T08:14:50,650 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/C in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:50,650 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/83889a7ef56949f7bbb395dea554bcdb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/9e7e9095fd7d4d3eaff895a06f08dc91, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/1436c99689164124b8607a746d46fcd4] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=36.3 K 2024-12-08T08:14:50,651 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 83889a7ef56949f7bbb395dea554bcdb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733645685339 2024-12-08T08:14:50,651 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e7e9095fd7d4d3eaff895a06f08dc91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733645686502 2024-12-08T08:14:50,651 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 1436c99689164124b8607a746d46fcd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733645688696 2024-12-08T08:14:50,658 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#C#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:50,658 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/55b3dd36b33545c7bcf7c6858f7f2195 is 50, key is test_row_0/C:col10/1733645688696/Put/seqid=0 2024-12-08T08:14:50,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742025_1201 (size=12949) 2024-12-08T08:14:50,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645750691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:50,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645750700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:50,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645750900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:50,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645750910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:50,918 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:50,922 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084fdac473dd6f4d539e49ecca66b502a2_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084fdac473dd6f4d539e49ecca66b502a2_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:50,923 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/7c5fd04bd2ab44279f6639c696c5901d, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:50,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/7c5fd04bd2ab44279f6639c696c5901d is 175, key is test_row_0/A:col10/1733645689353/Put/seqid=0 2024-12-08T08:14:50,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742026_1202 (size=39949) 2024-12-08T08:14:51,055 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/3c01c664367d433bb215838be2d7f7e2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/3c01c664367d433bb215838be2d7f7e2 2024-12-08T08:14:51,060 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/A of 45f279431c7ebbce8a61010c6b97623a into 3c01c664367d433bb215838be2d7f7e2(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:51,060 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:51,060 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/A, priority=13, startTime=1733645690222; duration=0sec 2024-12-08T08:14:51,060 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:51,060 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:A 2024-12-08T08:14:51,067 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/55b3dd36b33545c7bcf7c6858f7f2195 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/55b3dd36b33545c7bcf7c6858f7f2195 2024-12-08T08:14:51,071 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/C of 45f279431c7ebbce8a61010c6b97623a into 55b3dd36b33545c7bcf7c6858f7f2195(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:51,071 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:51,071 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/C, priority=13, startTime=1733645690222; duration=0sec 2024-12-08T08:14:51,071 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:51,071 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:C 2024-12-08T08:14:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T08:14:51,081 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:51,081 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-08T08:14:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T08:14:51,083 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:51,084 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:51,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T08:14:51,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645751211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:51,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645751220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:51,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T08:14:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:51,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:51,328 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/7c5fd04bd2ab44279f6639c696c5901d 2024-12-08T08:14:51,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/3fee7eb3b11f4997bb6fb444f7254594 is 50, key is test_row_0/B:col10/1733645689353/Put/seqid=0 2024-12-08T08:14:51,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742027_1203 (size=12301) 2024-12-08T08:14:51,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/3fee7eb3b11f4997bb6fb444f7254594 2024-12-08T08:14:51,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/15b335183f254d3ca028739bcee950b9 is 50, key is test_row_0/C:col10/1733645689353/Put/seqid=0 2024-12-08T08:14:51,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742028_1204 (size=12301) 2024-12-08T08:14:51,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T08:14:51,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:51,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:51,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T08:14:51,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T08:14:51,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:51,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:51,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:51,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:51,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:51,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T08:14:51,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:51,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:51,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:14:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T08:14:51,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645751730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:51,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:51,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645751730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:51,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:51,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:51,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:51,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:51,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:51,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/15b335183f254d3ca028739bcee950b9 2024-12-08T08:14:51,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/7c5fd04bd2ab44279f6639c696c5901d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/7c5fd04bd2ab44279f6639c696c5901d 2024-12-08T08:14:51,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/7c5fd04bd2ab44279f6639c696c5901d, entries=200, sequenceid=290, filesize=39.0 K 2024-12-08T08:14:51,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/3fee7eb3b11f4997bb6fb444f7254594 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3fee7eb3b11f4997bb6fb444f7254594 2024-12-08T08:14:51,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3fee7eb3b11f4997bb6fb444f7254594, entries=150, sequenceid=290, filesize=12.0 K 2024-12-08T08:14:51,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/15b335183f254d3ca028739bcee950b9 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/15b335183f254d3ca028739bcee950b9 2024-12-08T08:14:51,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/15b335183f254d3ca028739bcee950b9, entries=150, sequenceid=290, filesize=12.0 K 2024-12-08T08:14:51,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 45f279431c7ebbce8a61010c6b97623a in 1293ms, sequenceid=290, compaction requested=false 2024-12-08T08:14:51,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:51,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T08:14:51,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:51,848 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-08T08:14:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:51,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d18b0d21c35b46529c9541183c30dce4_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645690582/Put/seqid=0 2024-12-08T08:14:51,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742029_1205 (size=12454) 2024-12-08T08:14:51,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:51,868 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d18b0d21c35b46529c9541183c30dce4_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d18b0d21c35b46529c9541183c30dce4_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:51,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/83b2de9830ff4cd482dc93a814618463, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:51,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/83b2de9830ff4cd482dc93a814618463 is 175, key is test_row_0/A:col10/1733645690582/Put/seqid=0 2024-12-08T08:14:51,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742030_1206 (size=31255) 2024-12-08T08:14:51,877 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/83b2de9830ff4cd482dc93a814618463 2024-12-08T08:14:51,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/69d97adf6a5743c2b8384836c8a2c2ff is 50, key is test_row_0/B:col10/1733645690582/Put/seqid=0 2024-12-08T08:14:51,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742031_1207 (size=12301) 2024-12-08T08:14:51,898 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/69d97adf6a5743c2b8384836c8a2c2ff 2024-12-08T08:14:51,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/f3420548a3dd449e88aef839489e7069 is 50, key is test_row_0/C:col10/1733645690582/Put/seqid=0 2024-12-08T08:14:51,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742032_1208 (size=12301) 2024-12-08T08:14:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T08:14:52,319 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/f3420548a3dd449e88aef839489e7069 2024-12-08T08:14:52,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/83b2de9830ff4cd482dc93a814618463 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/83b2de9830ff4cd482dc93a814618463 2024-12-08T08:14:52,353 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/83b2de9830ff4cd482dc93a814618463, entries=150, sequenceid=327, filesize=30.5 K 2024-12-08T08:14:52,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/69d97adf6a5743c2b8384836c8a2c2ff as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/69d97adf6a5743c2b8384836c8a2c2ff 2024-12-08T08:14:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,358 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/69d97adf6a5743c2b8384836c8a2c2ff, entries=150, sequenceid=327, filesize=12.0 K 2024-12-08T08:14:52,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/f3420548a3dd449e88aef839489e7069 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/f3420548a3dd449e88aef839489e7069 2024-12-08T08:14:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,363 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/f3420548a3dd449e88aef839489e7069, entries=150, sequenceid=327, filesize=12.0 K 2024-12-08T08:14:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,364 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=0 B/0 for 45f279431c7ebbce8a61010c6b97623a in 516ms, sequenceid=327, compaction requested=true 2024-12-08T08:14:52,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:52,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:52,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-08T08:14:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-08T08:14:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-08T08:14:52,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2820 sec 2024-12-08T08:14:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.2870 sec 2024-12-08T08:14:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:52,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:14:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:52,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:52,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:52,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:52,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:52,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089c6a741e7e56493e99960e4e2cc3d1e3_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645692813/Put/seqid=0 2024-12-08T08:14:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742033_1209 (size=17534) 2024-12-08T08:14:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:52,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645692888, max timestamp=1733645692904Num uniques 859; 2024-12-08T08:14:52,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:14:52,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645692888, max timestamp=1733645692904Num uniques 859; 2024-12-08T08:14:52,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:14:52,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645692888, max timestamp=1733645692904Num uniques 859; 2024-12-08T08:14:52,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:14:52,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645752949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:52,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:52,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:52,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645752953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:52,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:52,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:52,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645753061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:53,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645753070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:53,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:53,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T08:14:53,230 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:53,231 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-08T08:14:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T08:14:53,232 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:53,233 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:53,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:53,243 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:53,246 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089c6a741e7e56493e99960e4e2cc3d1e3_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089c6a741e7e56493e99960e4e2cc3d1e3_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:53,247 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/353d02ac57d54178bd9d2ebd9f55da4e, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:53,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/353d02ac57d54178bd9d2ebd9f55da4e is 175, key is test_row_0/A:col10/1733645692813/Put/seqid=0 2024-12-08T08:14:53,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742034_1210 (size=48635) 2024-12-08T08:14:53,253 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/353d02ac57d54178bd9d2ebd9f55da4e 2024-12-08T08:14:53,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/1d45e3e0f7124de5b7768974d26827cc is 50, key is test_row_0/B:col10/1733645692813/Put/seqid=0 2024-12-08T08:14:53,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742035_1211 (size=9857) 2024-12-08T08:14:53,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645753283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:53,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645753284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:53,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/1d45e3e0f7124de5b7768974d26827cc 2024-12-08T08:14:53,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/f87cb50391a04079be8d70ea62f70e76 is 50, key is test_row_0/C:col10/1733645692813/Put/seqid=0 2024-12-08T08:14:53,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742036_1212 (size=9857) 2024-12-08T08:14:53,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/f87cb50391a04079be8d70ea62f70e76 2024-12-08T08:14:53,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/353d02ac57d54178bd9d2ebd9f55da4e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/353d02ac57d54178bd9d2ebd9f55da4e 2024-12-08T08:14:53,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/353d02ac57d54178bd9d2ebd9f55da4e, entries=250, sequenceid=337, filesize=47.5 K 2024-12-08T08:14:53,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/1d45e3e0f7124de5b7768974d26827cc as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/1d45e3e0f7124de5b7768974d26827cc 2024-12-08T08:14:53,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/1d45e3e0f7124de5b7768974d26827cc, entries=100, sequenceid=337, filesize=9.6 K 2024-12-08T08:14:53,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/f87cb50391a04079be8d70ea62f70e76 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/f87cb50391a04079be8d70ea62f70e76 2024-12-08T08:14:53,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/f87cb50391a04079be8d70ea62f70e76, entries=100, sequenceid=337, filesize=9.6 K 2024-12-08T08:14:53,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 45f279431c7ebbce8a61010c6b97623a in 524ms, sequenceid=337, compaction requested=true 2024-12-08T08:14:53,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:53,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:53,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:53,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:53,338 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:53,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:53,338 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:53,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:53,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:53,339 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:53,339 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:53,339 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/A is initiating minor compaction (all files) 2024-12-08T08:14:53,340 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/B is initiating minor compaction (all files) 2024-12-08T08:14:53,340 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/A in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:53,340 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/B in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:53,340 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/3c01c664367d433bb215838be2d7f7e2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/7c5fd04bd2ab44279f6639c696c5901d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/83b2de9830ff4cd482dc93a814618463, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/353d02ac57d54178bd9d2ebd9f55da4e] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=148.2 K 2024-12-08T08:14:53,340 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ca657c413d894313a94695c09985f810, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3fee7eb3b11f4997bb6fb444f7254594, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/69d97adf6a5743c2b8384836c8a2c2ff, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/1d45e3e0f7124de5b7768974d26827cc] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=46.3 K 2024-12-08T08:14:53,340 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:53,340 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/3c01c664367d433bb215838be2d7f7e2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/7c5fd04bd2ab44279f6639c696c5901d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/83b2de9830ff4cd482dc93a814618463, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/353d02ac57d54178bd9d2ebd9f55da4e] 2024-12-08T08:14:53,340 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting ca657c413d894313a94695c09985f810, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733645688696 2024-12-08T08:14:53,340 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c01c664367d433bb215838be2d7f7e2, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733645688696 2024-12-08T08:14:53,340 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fee7eb3b11f4997bb6fb444f7254594, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733645689344 2024-12-08T08:14:53,340 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c5fd04bd2ab44279f6639c696c5901d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733645689344 2024-12-08T08:14:53,341 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 69d97adf6a5743c2b8384836c8a2c2ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733645690582 2024-12-08T08:14:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T08:14:53,342 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83b2de9830ff4cd482dc93a814618463, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733645690582 2024-12-08T08:14:53,342 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d45e3e0f7124de5b7768974d26827cc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733645692802 2024-12-08T08:14:53,343 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 353d02ac57d54178bd9d2ebd9f55da4e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733645692786 2024-12-08T08:14:53,350 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:53,352 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:53,352 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/48b0d2abdfa64ef39418db56f8c1c9c1 is 50, key is test_row_0/B:col10/1733645692813/Put/seqid=0 2024-12-08T08:14:53,361 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412086092c95d3966459d8056d49b29e77dbc_45f279431c7ebbce8a61010c6b97623a store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:53,364 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412086092c95d3966459d8056d49b29e77dbc_45f279431c7ebbce8a61010c6b97623a, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:53,364 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086092c95d3966459d8056d49b29e77dbc_45f279431c7ebbce8a61010c6b97623a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:53,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742037_1213 (size=13085) 2024-12-08T08:14:53,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742038_1214 (size=4469) 2024-12-08T08:14:53,385 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/48b0d2abdfa64ef39418db56f8c1c9c1 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/48b0d2abdfa64ef39418db56f8c1c9c1 2024-12-08T08:14:53,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T08:14:53,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:53,385 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-08T08:14:53,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:53,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:53,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:53,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:53,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:53,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:53,396 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/B of 45f279431c7ebbce8a61010c6b97623a into 48b0d2abdfa64ef39418db56f8c1c9c1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:53,396 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:53,396 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/B, priority=12, startTime=1733645693338; duration=0sec 2024-12-08T08:14:53,396 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:53,396 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:B 2024-12-08T08:14:53,396 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:53,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120885283bb08a38461e9826c718c4027632_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645692929/Put/seqid=0 2024-12-08T08:14:53,397 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:53,398 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/C is initiating minor compaction (all files) 2024-12-08T08:14:53,398 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/C in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:53,398 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/55b3dd36b33545c7bcf7c6858f7f2195, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/15b335183f254d3ca028739bcee950b9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/f3420548a3dd449e88aef839489e7069, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/f87cb50391a04079be8d70ea62f70e76] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=46.3 K 2024-12-08T08:14:53,398 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 55b3dd36b33545c7bcf7c6858f7f2195, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733645688696 2024-12-08T08:14:53,398 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 15b335183f254d3ca028739bcee950b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733645689344 2024-12-08T08:14:53,399 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting f3420548a3dd449e88aef839489e7069, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733645690582 2024-12-08T08:14:53,399 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting f87cb50391a04079be8d70ea62f70e76, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733645692802 2024-12-08T08:14:53,410 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:53,411 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/4cd58941a5ab47b1a46fa48e4d2bbac8 is 50, key is test_row_0/C:col10/1733645692813/Put/seqid=0 2024-12-08T08:14:53,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742039_1215 (size=12454) 2024-12-08T08:14:53,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742040_1216 (size=13085) 2024-12-08T08:14:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T08:14:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:53,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:53,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645753612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645753614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:53,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645753721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:53,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645753721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:53,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42852 deadline: 1733645753760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:53,763 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 45f279431c7ebbce8a61010c6b97623a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18297 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:14:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:53,784 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#A#compaction#175 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:53,784 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/7d5e17173893448898907705ed704d9b is 175, key is test_row_0/A:col10/1733645692813/Put/seqid=0 2024-12-08T08:14:53,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742041_1217 (size=32146) 2024-12-08T08:14:53,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:53,832 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120885283bb08a38461e9826c718c4027632_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120885283bb08a38461e9826c718c4027632_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:53,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/caca4e9b37e94ce8921a8205fdeb4a6e, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:53,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/caca4e9b37e94ce8921a8205fdeb4a6e is 175, key is test_row_0/A:col10/1733645692929/Put/seqid=0 2024-12-08T08:14:53,837 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/4cd58941a5ab47b1a46fa48e4d2bbac8 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/4cd58941a5ab47b1a46fa48e4d2bbac8 2024-12-08T08:14:53,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742042_1218 (size=31255) 2024-12-08T08:14:53,842 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/caca4e9b37e94ce8921a8205fdeb4a6e 2024-12-08T08:14:53,846 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/C of 45f279431c7ebbce8a61010c6b97623a into 4cd58941a5ab47b1a46fa48e4d2bbac8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:53,846 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:53,846 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/C, priority=12, startTime=1733645693338; duration=0sec 2024-12-08T08:14:53,846 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:53,846 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:C 2024-12-08T08:14:53,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/cbd1226e1e7d4651b1dc566016de8a98 is 50, key is test_row_0/B:col10/1733645692929/Put/seqid=0 2024-12-08T08:14:53,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742043_1219 (size=12301) 2024-12-08T08:14:53,860 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/cbd1226e1e7d4651b1dc566016de8a98 2024-12-08T08:14:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T08:14:53,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/ce69fd0573e543958def59d405576300 is 50, key is test_row_0/C:col10/1733645692929/Put/seqid=0 2024-12-08T08:14:53,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742044_1220 (size=12301) 2024-12-08T08:14:53,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645753930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645753930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:54,196 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/7d5e17173893448898907705ed704d9b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/7d5e17173893448898907705ed704d9b 2024-12-08T08:14:54,204 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/A of 45f279431c7ebbce8a61010c6b97623a into 7d5e17173893448898907705ed704d9b(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:54,205 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:54,205 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/A, priority=12, startTime=1733645693338; duration=0sec 2024-12-08T08:14:54,205 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:54,205 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:A 2024-12-08T08:14:54,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645754241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:54,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645754241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:54,282 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/ce69fd0573e543958def59d405576300 2024-12-08T08:14:54,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/caca4e9b37e94ce8921a8205fdeb4a6e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/caca4e9b37e94ce8921a8205fdeb4a6e 2024-12-08T08:14:54,292 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/caca4e9b37e94ce8921a8205fdeb4a6e, entries=150, sequenceid=373, filesize=30.5 K 2024-12-08T08:14:54,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/cbd1226e1e7d4651b1dc566016de8a98 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/cbd1226e1e7d4651b1dc566016de8a98 2024-12-08T08:14:54,299 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/cbd1226e1e7d4651b1dc566016de8a98, entries=150, sequenceid=373, filesize=12.0 K 2024-12-08T08:14:54,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/ce69fd0573e543958def59d405576300 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/ce69fd0573e543958def59d405576300 2024-12-08T08:14:54,306 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/ce69fd0573e543958def59d405576300, entries=150, sequenceid=373, filesize=12.0 K 2024-12-08T08:14:54,307 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 45f279431c7ebbce8a61010c6b97623a in 921ms, sequenceid=373, compaction requested=false 2024-12-08T08:14:54,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:54,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:54,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-08T08:14:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-08T08:14:54,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-08T08:14:54,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0750 sec 2024-12-08T08:14:54,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.0810 sec 2024-12-08T08:14:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T08:14:54,371 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:54,371 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:14:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-08T08:14:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T08:14:54,373 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:14:54,373 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:14:54,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:14:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T08:14:54,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T08:14:54,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:54,526 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:14:54,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:54,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:54,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:54,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:54,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:54,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:14:54,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080b5e32e0d4794803ae7c05c8426275af_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645693609/Put/seqid=0 2024-12-08T08:14:54,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742045_1221 (size=12454) 2024-12-08T08:14:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T08:14:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:54,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. as already flushing 2024-12-08T08:14:54,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645694769, max timestamp=1733645694788Num uniques 859; 2024-12-08T08:14:54,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:14:54,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645694769, max timestamp=1733645694788Num uniques 859; 2024-12-08T08:14:54,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:14:54,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645694769, max timestamp=1733645694788Num uniques 859; 2024-12-08T08:14:54,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:14:54,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:54,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645754825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645754823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:54,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:54,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645754942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645754942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:54,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:54,962 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080b5e32e0d4794803ae7c05c8426275af_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080b5e32e0d4794803ae7c05c8426275af_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:54,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/b14bedb6a34a4cdd99743ecace134774, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:54,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/b14bedb6a34a4cdd99743ecace134774 is 175, key is test_row_0/A:col10/1733645693609/Put/seqid=0 2024-12-08T08:14:54,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742046_1222 (size=31255) 2024-12-08T08:14:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T08:14:55,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645755150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:55,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645755150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:55,298 INFO [Thread-620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-620. 2024-12-08T08:14:55,298 DEBUG [Thread-620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:55,298 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:55,298 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:55,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:55,301 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-12-08T08:14:55,301 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:55,301 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:55,301 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:55,301 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:55,303 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-12-08T08:14:55,303 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:55,303 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:55,303 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:55,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:55,305 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-12-08T08:14:55,305 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:55,305 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:55,305 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:55,305 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:55,369 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/b14bedb6a34a4cdd99743ecace134774 2024-12-08T08:14:55,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/714b8afeace24ebc9f3251aebec77d4f is 50, key is test_row_0/B:col10/1733645693609/Put/seqid=0 2024-12-08T08:14:55,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742047_1223 (size=12301) 2024-12-08T08:14:55,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:55,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645755462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645755462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:55,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:55,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T08:14:55,791 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/714b8afeace24ebc9f3251aebec77d4f 2024-12-08T08:14:55,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/a42d24620d5345358ba6594e38347e30 is 50, key is test_row_0/C:col10/1733645693609/Put/seqid=0 2024-12-08T08:14:55,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742048_1224 (size=12301) 2024-12-08T08:14:55,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:55,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:14:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42836 deadline: 1733645755972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42820 deadline: 1733645755972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 2024-12-08T08:14:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:14:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:14:56,212 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/a42d24620d5345358ba6594e38347e30 2024-12-08T08:14:56,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/b14bedb6a34a4cdd99743ecace134774 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/b14bedb6a34a4cdd99743ecace134774 2024-12-08T08:14:56,229 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/b14bedb6a34a4cdd99743ecace134774, entries=150, sequenceid=386, filesize=30.5 K 2024-12-08T08:14:56,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/714b8afeace24ebc9f3251aebec77d4f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/714b8afeace24ebc9f3251aebec77d4f 2024-12-08T08:14:56,236 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/714b8afeace24ebc9f3251aebec77d4f, entries=150, sequenceid=386, filesize=12.0 K 2024-12-08T08:14:56,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/a42d24620d5345358ba6594e38347e30 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/a42d24620d5345358ba6594e38347e30 2024-12-08T08:14:56,241 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/a42d24620d5345358ba6594e38347e30, entries=150, sequenceid=386, filesize=12.0 K 2024-12-08T08:14:56,242 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 45f279431c7ebbce8a61010c6b97623a in 1716ms, sequenceid=386, compaction requested=true 2024-12-08T08:14:56,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:56,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:56,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-08T08:14:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-08T08:14:56,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-08T08:14:56,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8700 sec 2024-12-08T08:14:56,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.8740 sec 2024-12-08T08:14:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T08:14:56,523 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:14:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:56,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-08T08:14:56,986 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-12-08T08:14:56,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:14:56,986 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:56,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:56,986 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:56,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:14:56,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:56,986 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:56,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:14:56,986 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:56,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:14:56,993 INFO [Thread-617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-617. 2024-12-08T08:14:56,994 DEBUG [Thread-617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:56,994 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:56,994 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:56,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:56,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b7740cc0e32146279a4237c43a2661c5_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_0/A:col10/1733645694823/Put/seqid=0 2024-12-08T08:14:57,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742049_1225 (size=12454) 2024-12-08T08:14:57,403 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:14:57,467 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b7740cc0e32146279a4237c43a2661c5_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b7740cc0e32146279a4237c43a2661c5_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:14:57,470 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/181d03e1dfc44073956a31246a9fc6ff, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:57,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/181d03e1dfc44073956a31246a9fc6ff is 175, key is test_row_0/A:col10/1733645694823/Put/seqid=0 2024-12-08T08:14:57,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742050_1226 (size=31255) 2024-12-08T08:14:57,879 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/181d03e1dfc44073956a31246a9fc6ff 2024-12-08T08:14:57,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/e087eea045a749d0abc0b62dac51c3fc is 50, key is test_row_0/B:col10/1733645694823/Put/seqid=0 2024-12-08T08:14:57,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742051_1227 (size=12301) 2024-12-08T08:14:58,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/e087eea045a749d0abc0b62dac51c3fc 2024-12-08T08:14:58,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/861b424c480847e6a2a170187feae019 is 50, key is test_row_0/C:col10/1733645694823/Put/seqid=0 2024-12-08T08:14:58,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742052_1228 (size=12301) 2024-12-08T08:14:58,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/861b424c480847e6a2a170187feae019 2024-12-08T08:14:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/181d03e1dfc44073956a31246a9fc6ff as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/181d03e1dfc44073956a31246a9fc6ff 2024-12-08T08:14:58,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/181d03e1dfc44073956a31246a9fc6ff, entries=150, sequenceid=423, filesize=30.5 K 2024-12-08T08:14:58,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/e087eea045a749d0abc0b62dac51c3fc as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e087eea045a749d0abc0b62dac51c3fc 2024-12-08T08:14:58,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e087eea045a749d0abc0b62dac51c3fc, entries=150, sequenceid=423, filesize=12.0 K 2024-12-08T08:14:58,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/861b424c480847e6a2a170187feae019 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/861b424c480847e6a2a170187feae019 2024-12-08T08:14:58,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/861b424c480847e6a2a170187feae019, entries=150, sequenceid=423, filesize=12.0 K 2024-12-08T08:14:58,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=6.71 KB/6870 for 45f279431c7ebbce8a61010c6b97623a in 1754ms, sequenceid=423, compaction requested=true 2024-12-08T08:14:58,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:58,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:14:58,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:58,740 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:58,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:14:58,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:58,740 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:58,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f279431c7ebbce8a61010c6b97623a:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:14:58,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:58,741 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125911 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:58,741 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/A is initiating minor compaction (all files) 2024-12-08T08:14:58,741 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:58,741 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/A in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:58,741 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/B is initiating minor compaction (all files) 2024-12-08T08:14:58,741 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/B in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:58,741 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/7d5e17173893448898907705ed704d9b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/caca4e9b37e94ce8921a8205fdeb4a6e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/b14bedb6a34a4cdd99743ecace134774, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/181d03e1dfc44073956a31246a9fc6ff] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=123.0 K 2024-12-08T08:14:58,741 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:58,741 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/48b0d2abdfa64ef39418db56f8c1c9c1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/cbd1226e1e7d4651b1dc566016de8a98, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/714b8afeace24ebc9f3251aebec77d4f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e087eea045a749d0abc0b62dac51c3fc] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=48.8 K 2024-12-08T08:14:58,741 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/7d5e17173893448898907705ed704d9b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/caca4e9b37e94ce8921a8205fdeb4a6e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/b14bedb6a34a4cdd99743ecace134774, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/181d03e1dfc44073956a31246a9fc6ff] 2024-12-08T08:14:58,741 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d5e17173893448898907705ed704d9b, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733645690584 2024-12-08T08:14:58,741 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 48b0d2abdfa64ef39418db56f8c1c9c1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733645690584 2024-12-08T08:14:58,742 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting caca4e9b37e94ce8921a8205fdeb4a6e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733645692929 2024-12-08T08:14:58,742 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting cbd1226e1e7d4651b1dc566016de8a98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733645692929 2024-12-08T08:14:58,742 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting b14bedb6a34a4cdd99743ecace134774, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733645693609 2024-12-08T08:14:58,742 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 714b8afeace24ebc9f3251aebec77d4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733645693609 2024-12-08T08:14:58,742 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 181d03e1dfc44073956a31246a9fc6ff, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733645694822 2024-12-08T08:14:58,742 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting e087eea045a749d0abc0b62dac51c3fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733645694822 2024-12-08T08:14:58,752 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#B#compaction#186 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:58,752 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/e12d6a4206c743599f90faf15d7f29bb is 50, key is test_row_0/B:col10/1733645694823/Put/seqid=0 2024-12-08T08:14:58,753 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:58,756 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120894297678310a410aa96a63a10a88c4b1_45f279431c7ebbce8a61010c6b97623a store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:58,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742053_1229 (size=13221) 2024-12-08T08:14:58,774 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120894297678310a410aa96a63a10a88c4b1_45f279431c7ebbce8a61010c6b97623a, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:58,775 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120894297678310a410aa96a63a10a88c4b1_45f279431c7ebbce8a61010c6b97623a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:14:58,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742054_1230 (size=4469) 2024-12-08T08:14:58,942 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-12-08T08:14:58,942 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:58,942 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:58,942 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:58,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:58,952 INFO [Thread-609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-609. 2024-12-08T08:14:58,952 DEBUG [Thread-609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:14:58,952 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:58,952 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:14:58,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:14:59,171 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/e12d6a4206c743599f90faf15d7f29bb as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e12d6a4206c743599f90faf15d7f29bb 2024-12-08T08:14:59,178 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/B of 45f279431c7ebbce8a61010c6b97623a into e12d6a4206c743599f90faf15d7f29bb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:59,178 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:59,178 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/B, priority=12, startTime=1733645698740; duration=0sec 2024-12-08T08:14:59,178 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:14:59,178 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:B 2024-12-08T08:14:59,178 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:14:59,179 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#A#compaction#187 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:59,180 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:14:59,180 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 45f279431c7ebbce8a61010c6b97623a/C is initiating minor compaction (all files) 2024-12-08T08:14:59,180 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f279431c7ebbce8a61010c6b97623a/C in TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:14:59,180 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/4cd58941a5ab47b1a46fa48e4d2bbac8, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/ce69fd0573e543958def59d405576300, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/a42d24620d5345358ba6594e38347e30, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/861b424c480847e6a2a170187feae019] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp, totalSize=48.8 K 2024-12-08T08:14:59,180 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/f710f47fefe144e58068d70a601946f7 is 175, key is test_row_0/A:col10/1733645694823/Put/seqid=0 2024-12-08T08:14:59,181 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cd58941a5ab47b1a46fa48e4d2bbac8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733645690584 2024-12-08T08:14:59,181 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting ce69fd0573e543958def59d405576300, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733645692929 2024-12-08T08:14:59,181 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting a42d24620d5345358ba6594e38347e30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733645693609 2024-12-08T08:14:59,182 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 861b424c480847e6a2a170187feae019, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733645694822 2024-12-08T08:14:59,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742055_1231 (size=32175) 2024-12-08T08:14:59,192 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f279431c7ebbce8a61010c6b97623a#C#compaction#188 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:14:59,192 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/33d97e14695d417ab81c1482ad71fde4 is 50, key is test_row_0/C:col10/1733645694823/Put/seqid=0 2024-12-08T08:14:59,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742056_1232 (size=13221) 2024-12-08T08:14:59,201 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/33d97e14695d417ab81c1482ad71fde4 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/33d97e14695d417ab81c1482ad71fde4 2024-12-08T08:14:59,206 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/C of 45f279431c7ebbce8a61010c6b97623a into 33d97e14695d417ab81c1482ad71fde4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:59,206 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:59,206 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/C, priority=12, startTime=1733645698740; duration=0sec 2024-12-08T08:14:59,206 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:59,206 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:C 2024-12-08T08:14:59,597 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/f710f47fefe144e58068d70a601946f7 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/f710f47fefe144e58068d70a601946f7 2024-12-08T08:14:59,602 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f279431c7ebbce8a61010c6b97623a/A of 45f279431c7ebbce8a61010c6b97623a into f710f47fefe144e58068d70a601946f7(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:14:59,602 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f279431c7ebbce8a61010c6b97623a: 2024-12-08T08:14:59,602 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., storeName=45f279431c7ebbce8a61010c6b97623a/A, priority=12, startTime=1733645698740; duration=0sec 2024-12-08T08:14:59,602 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:14:59,602 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f279431c7ebbce8a61010c6b97623a:A 2024-12-08T08:15:00,407 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T08:15:03,802 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-12-08T08:15:03,802 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:03,802 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:03,803 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:03,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 130 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3916 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4009 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1726 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5177 rows 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1716 2024-12-08T08:15:03,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5148 rows 2024-12-08T08:15:03,803 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T08:15:03,803 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T08:15:03,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:03,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:03,804 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:03,804 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-08T08:15:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T08:15:03,808 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645703807"}]},"ts":"1733645703807"} 2024-12-08T08:15:03,809 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T08:15:03,809 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T08:15:03,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T08:15:03,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, UNASSIGN}] 2024-12-08T08:15:03,812 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, UNASSIGN 2024-12-08T08:15:03,813 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=45f279431c7ebbce8a61010c6b97623a, regionState=CLOSING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:15:03,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, UNASSIGN because future has completed 2024-12-08T08:15:03,815 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T08:15:03,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:15:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T08:15:03,968 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:03,968 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T08:15:03,968 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 45f279431c7ebbce8a61010c6b97623a, disabling compactions & flushes 2024-12-08T08:15:03,968 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:15:03,968 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:15:03,968 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. after waiting 0 ms 2024-12-08T08:15:03,968 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:15:03,968 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 45f279431c7ebbce8a61010c6b97623a 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-08T08:15:03,968 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=A 2024-12-08T08:15:03,968 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:03,968 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=B 2024-12-08T08:15:03,968 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:03,969 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f279431c7ebbce8a61010c6b97623a, store=C 2024-12-08T08:15:03,969 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:03,975 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c3db1f80ef834615ac8ea3df6662d27a_45f279431c7ebbce8a61010c6b97623a is 50, key is test_row_1/A:col10/1733645703801/Put/seqid=0 2024-12-08T08:15:03,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742057_1233 (size=9914) 2024-12-08T08:15:03,983 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:03,988 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c3db1f80ef834615ac8ea3df6662d27a_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c3db1f80ef834615ac8ea3df6662d27a_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:03,989 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/358f51b5930640299c6bd285945a8fb3, store: [table=TestAcidGuarantees family=A region=45f279431c7ebbce8a61010c6b97623a] 2024-12-08T08:15:03,989 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/358f51b5930640299c6bd285945a8fb3 is 175, key is test_row_1/A:col10/1733645703801/Put/seqid=0 2024-12-08T08:15:03,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742058_1234 (size=22561) 2024-12-08T08:15:03,995 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/358f51b5930640299c6bd285945a8fb3 2024-12-08T08:15:04,004 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/9ba1c1d750cb4514a837f864686e4525 is 50, key is test_row_1/B:col10/1733645703801/Put/seqid=0 2024-12-08T08:15:04,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742059_1235 (size=9857) 2024-12-08T08:15:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T08:15:04,420 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/9ba1c1d750cb4514a837f864686e4525 2024-12-08T08:15:04,427 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/13c41f28b0954629b44a3b7fcfe87dfb is 50, key is test_row_1/C:col10/1733645703801/Put/seqid=0 2024-12-08T08:15:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T08:15:04,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742060_1236 (size=9857) 2024-12-08T08:15:04,831 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/13c41f28b0954629b44a3b7fcfe87dfb 2024-12-08T08:15:04,836 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/A/358f51b5930640299c6bd285945a8fb3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/358f51b5930640299c6bd285945a8fb3 2024-12-08T08:15:04,840 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/358f51b5930640299c6bd285945a8fb3, entries=100, sequenceid=433, filesize=22.0 K 2024-12-08T08:15:04,840 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/B/9ba1c1d750cb4514a837f864686e4525 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/9ba1c1d750cb4514a837f864686e4525 2024-12-08T08:15:04,844 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/9ba1c1d750cb4514a837f864686e4525, entries=100, sequenceid=433, filesize=9.6 K 2024-12-08T08:15:04,845 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/.tmp/C/13c41f28b0954629b44a3b7fcfe87dfb as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/13c41f28b0954629b44a3b7fcfe87dfb 2024-12-08T08:15:04,849 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/13c41f28b0954629b44a3b7fcfe87dfb, entries=100, sequenceid=433, filesize=9.6 K 2024-12-08T08:15:04,850 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 45f279431c7ebbce8a61010c6b97623a in 882ms, sequenceid=433, compaction requested=false 2024-12-08T08:15:04,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e521e65d8aec4f4b8ced26758a426883, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1c79e4b3037e4762af97042fd2750434, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/c6c46db862e6431a96d2aed8f19606a3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/a0150ee9556a4ee89b50fe7c1f4abe32, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/00cceff5669047fc8b1598b138d0132d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8ccece8491ce40f28789875cabe021fd, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/b201e8f7a9e4473da7a3c3466158291f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1a88df9f404449b597cbf06f1a8ab3f5, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/04142441a66c4ac0959b7bee82980aac, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e08e4e79e7b24b07bfc461aed4ccbf50, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/5d82fa2430d447d88b36fe5f0dcf5ab1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/fd2964037d9f4693ab1ce023ef77f130, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/60e883e7d0f8439981151094ef7e68f5, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8701d62816f24fc4945116c5fb5ba31c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/2b635dd917554df8931d7a3932035acc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/87664ffafdcb4cbc8e3f6dfb7840cd58, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/9ce6e6fb9edb4c22bee18d2ef4b71a4a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e3b111040d9c46c5adba015a0a6f20a0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/f6207fdd88cf41c6854180de3446799b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/3c01c664367d433bb215838be2d7f7e2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/7c5fd04bd2ab44279f6639c696c5901d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/83b2de9830ff4cd482dc93a814618463, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/353d02ac57d54178bd9d2ebd9f55da4e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/7d5e17173893448898907705ed704d9b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/caca4e9b37e94ce8921a8205fdeb4a6e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/b14bedb6a34a4cdd99743ecace134774, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/181d03e1dfc44073956a31246a9fc6ff] to archive 2024-12-08T08:15:04,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:15:04,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e521e65d8aec4f4b8ced26758a426883 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e521e65d8aec4f4b8ced26758a426883 2024-12-08T08:15:04,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1c79e4b3037e4762af97042fd2750434 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1c79e4b3037e4762af97042fd2750434 2024-12-08T08:15:04,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/c6c46db862e6431a96d2aed8f19606a3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/c6c46db862e6431a96d2aed8f19606a3 2024-12-08T08:15:04,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/a0150ee9556a4ee89b50fe7c1f4abe32 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/a0150ee9556a4ee89b50fe7c1f4abe32 2024-12-08T08:15:04,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/00cceff5669047fc8b1598b138d0132d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/00cceff5669047fc8b1598b138d0132d 2024-12-08T08:15:04,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8ccece8491ce40f28789875cabe021fd to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8ccece8491ce40f28789875cabe021fd 2024-12-08T08:15:04,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/b201e8f7a9e4473da7a3c3466158291f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/b201e8f7a9e4473da7a3c3466158291f 2024-12-08T08:15:04,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1a88df9f404449b597cbf06f1a8ab3f5 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/1a88df9f404449b597cbf06f1a8ab3f5 2024-12-08T08:15:04,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/04142441a66c4ac0959b7bee82980aac to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/04142441a66c4ac0959b7bee82980aac 2024-12-08T08:15:04,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e08e4e79e7b24b07bfc461aed4ccbf50 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e08e4e79e7b24b07bfc461aed4ccbf50 2024-12-08T08:15:04,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/5d82fa2430d447d88b36fe5f0dcf5ab1 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/5d82fa2430d447d88b36fe5f0dcf5ab1 2024-12-08T08:15:04,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/fd2964037d9f4693ab1ce023ef77f130 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/fd2964037d9f4693ab1ce023ef77f130 2024-12-08T08:15:04,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/60e883e7d0f8439981151094ef7e68f5 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/60e883e7d0f8439981151094ef7e68f5 2024-12-08T08:15:04,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8701d62816f24fc4945116c5fb5ba31c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/8701d62816f24fc4945116c5fb5ba31c 2024-12-08T08:15:04,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/2b635dd917554df8931d7a3932035acc to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/2b635dd917554df8931d7a3932035acc 2024-12-08T08:15:04,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/87664ffafdcb4cbc8e3f6dfb7840cd58 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/87664ffafdcb4cbc8e3f6dfb7840cd58 2024-12-08T08:15:04,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/9ce6e6fb9edb4c22bee18d2ef4b71a4a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/9ce6e6fb9edb4c22bee18d2ef4b71a4a 2024-12-08T08:15:04,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e3b111040d9c46c5adba015a0a6f20a0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/e3b111040d9c46c5adba015a0a6f20a0 2024-12-08T08:15:04,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/f6207fdd88cf41c6854180de3446799b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/f6207fdd88cf41c6854180de3446799b 2024-12-08T08:15:04,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/3c01c664367d433bb215838be2d7f7e2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/3c01c664367d433bb215838be2d7f7e2 2024-12-08T08:15:04,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/7c5fd04bd2ab44279f6639c696c5901d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/7c5fd04bd2ab44279f6639c696c5901d 2024-12-08T08:15:04,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/83b2de9830ff4cd482dc93a814618463 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/83b2de9830ff4cd482dc93a814618463 2024-12-08T08:15:04,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/353d02ac57d54178bd9d2ebd9f55da4e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/353d02ac57d54178bd9d2ebd9f55da4e 2024-12-08T08:15:04,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/7d5e17173893448898907705ed704d9b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/7d5e17173893448898907705ed704d9b 2024-12-08T08:15:04,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/caca4e9b37e94ce8921a8205fdeb4a6e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/caca4e9b37e94ce8921a8205fdeb4a6e 2024-12-08T08:15:04,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/b14bedb6a34a4cdd99743ecace134774 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/b14bedb6a34a4cdd99743ecace134774 2024-12-08T08:15:04,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/181d03e1dfc44073956a31246a9fc6ff to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/181d03e1dfc44073956a31246a9fc6ff 2024-12-08T08:15:04,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ae119dfbf25e4808bfdbc3c1595f8c2e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/b36b96e71e7a4111809d1331ce9182f1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ba9a756a6b2b4fbab26a373c8df95ad2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/aceabf0b310b4d06b86acb66b1cc6f92, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/b57fc11f2b174c2680391ffb5367aed7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/6d4e239c44374b7fb991a95e50f836fc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/8b0d8b8f4e92469c9cf771ca270a350b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/d81079f17e8d4e1a831b50490c49ed0c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e110f3bea6304628975469c8d99706ef, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/4451d95452954d78a8d7c034eecf1ad8, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3bc63148d5814449a544fee774ccb5a9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/871bc02bf7f64c999efaee90e66228dc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/94e5ae3d5f8b4a90b72a30dac6d8f334, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/50aace9e1d1e442587b198646c316c07, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/1413920a7bfe4377b5568bd99ecb1f28, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/263ccfb4df4942ca8512122be2c26246, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/4ecc3c3b039745a08c3fb55f612040f7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/f3ec4765b841414a9b6c90a4e24f56b8, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ca657c413d894313a94695c09985f810, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3162fd06373b4b6bb6c4baef98b83f58, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3fee7eb3b11f4997bb6fb444f7254594, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/69d97adf6a5743c2b8384836c8a2c2ff, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/48b0d2abdfa64ef39418db56f8c1c9c1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/1d45e3e0f7124de5b7768974d26827cc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/cbd1226e1e7d4651b1dc566016de8a98, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/714b8afeace24ebc9f3251aebec77d4f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e087eea045a749d0abc0b62dac51c3fc] to archive 2024-12-08T08:15:04,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:15:04,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ae119dfbf25e4808bfdbc3c1595f8c2e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ae119dfbf25e4808bfdbc3c1595f8c2e 2024-12-08T08:15:04,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/b36b96e71e7a4111809d1331ce9182f1 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/b36b96e71e7a4111809d1331ce9182f1 2024-12-08T08:15:04,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ba9a756a6b2b4fbab26a373c8df95ad2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ba9a756a6b2b4fbab26a373c8df95ad2 2024-12-08T08:15:04,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/aceabf0b310b4d06b86acb66b1cc6f92 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/aceabf0b310b4d06b86acb66b1cc6f92 2024-12-08T08:15:04,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/b57fc11f2b174c2680391ffb5367aed7 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/b57fc11f2b174c2680391ffb5367aed7 2024-12-08T08:15:04,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/6d4e239c44374b7fb991a95e50f836fc to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/6d4e239c44374b7fb991a95e50f836fc 2024-12-08T08:15:04,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/8b0d8b8f4e92469c9cf771ca270a350b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/8b0d8b8f4e92469c9cf771ca270a350b 2024-12-08T08:15:04,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/d81079f17e8d4e1a831b50490c49ed0c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/d81079f17e8d4e1a831b50490c49ed0c 2024-12-08T08:15:04,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e110f3bea6304628975469c8d99706ef to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e110f3bea6304628975469c8d99706ef 2024-12-08T08:15:04,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/4451d95452954d78a8d7c034eecf1ad8 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/4451d95452954d78a8d7c034eecf1ad8 2024-12-08T08:15:04,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3bc63148d5814449a544fee774ccb5a9 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3bc63148d5814449a544fee774ccb5a9 2024-12-08T08:15:04,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/871bc02bf7f64c999efaee90e66228dc to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/871bc02bf7f64c999efaee90e66228dc 2024-12-08T08:15:04,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/94e5ae3d5f8b4a90b72a30dac6d8f334 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/94e5ae3d5f8b4a90b72a30dac6d8f334 2024-12-08T08:15:04,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/50aace9e1d1e442587b198646c316c07 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/50aace9e1d1e442587b198646c316c07 2024-12-08T08:15:04,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/1413920a7bfe4377b5568bd99ecb1f28 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/1413920a7bfe4377b5568bd99ecb1f28 2024-12-08T08:15:04,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/263ccfb4df4942ca8512122be2c26246 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/263ccfb4df4942ca8512122be2c26246 2024-12-08T08:15:04,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/4ecc3c3b039745a08c3fb55f612040f7 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/4ecc3c3b039745a08c3fb55f612040f7 2024-12-08T08:15:04,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/f3ec4765b841414a9b6c90a4e24f56b8 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/f3ec4765b841414a9b6c90a4e24f56b8 2024-12-08T08:15:04,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ca657c413d894313a94695c09985f810 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/ca657c413d894313a94695c09985f810 2024-12-08T08:15:04,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3162fd06373b4b6bb6c4baef98b83f58 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3162fd06373b4b6bb6c4baef98b83f58 2024-12-08T08:15:04,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3fee7eb3b11f4997bb6fb444f7254594 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/3fee7eb3b11f4997bb6fb444f7254594 2024-12-08T08:15:04,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/69d97adf6a5743c2b8384836c8a2c2ff to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/69d97adf6a5743c2b8384836c8a2c2ff 2024-12-08T08:15:04,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/48b0d2abdfa64ef39418db56f8c1c9c1 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/48b0d2abdfa64ef39418db56f8c1c9c1 2024-12-08T08:15:04,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/1d45e3e0f7124de5b7768974d26827cc to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/1d45e3e0f7124de5b7768974d26827cc 2024-12-08T08:15:04,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/cbd1226e1e7d4651b1dc566016de8a98 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/cbd1226e1e7d4651b1dc566016de8a98 2024-12-08T08:15:04,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/714b8afeace24ebc9f3251aebec77d4f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/714b8afeace24ebc9f3251aebec77d4f 2024-12-08T08:15:04,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e087eea045a749d0abc0b62dac51c3fc to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e087eea045a749d0abc0b62dac51c3fc 2024-12-08T08:15:04,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/b0a04805ed8e440b8a5e6085271305cb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/573f0a9575384b7cb07ba3c726bb03d5, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/51e0e25cc4774806a4e4a082d6d9cea2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/a90454cabefa4797ad5ea8a12e223df1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/c0eaea04c5314f8ba8e355f4d9a8aa94, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/45a4fc15c8854d0882f187351dd3d38f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/e087830bf8c94cc287e85052414b23f9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/061ff3a311be417195326c051d79b92b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/19c6450281fc42e0b1f25a7e91dfb80e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/5ba03976e0de47629553d44625bb9f91, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/cdefbc56041340999764f22082a3337d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/4eab6745862a485e92111bf74644c910, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/ee045b44503e44b1b89e41d8f3c2cf27, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/cfe442c60b6e4d6db3f1eb815e9d6188, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/8d357626d1a341e796844fe03e53ed45, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/83889a7ef56949f7bbb395dea554bcdb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/c3ee9b417cbf4837a7aac4008ae5c989, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/9e7e9095fd7d4d3eaff895a06f08dc91, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/55b3dd36b33545c7bcf7c6858f7f2195, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/1436c99689164124b8607a746d46fcd4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/15b335183f254d3ca028739bcee950b9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/f3420548a3dd449e88aef839489e7069, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/4cd58941a5ab47b1a46fa48e4d2bbac8, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/f87cb50391a04079be8d70ea62f70e76, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/ce69fd0573e543958def59d405576300, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/a42d24620d5345358ba6594e38347e30, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/861b424c480847e6a2a170187feae019] to archive 2024-12-08T08:15:04,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:15:04,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/b0a04805ed8e440b8a5e6085271305cb to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/b0a04805ed8e440b8a5e6085271305cb 2024-12-08T08:15:04,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/573f0a9575384b7cb07ba3c726bb03d5 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/573f0a9575384b7cb07ba3c726bb03d5 2024-12-08T08:15:04,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/51e0e25cc4774806a4e4a082d6d9cea2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/51e0e25cc4774806a4e4a082d6d9cea2 2024-12-08T08:15:04,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/a90454cabefa4797ad5ea8a12e223df1 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/a90454cabefa4797ad5ea8a12e223df1 2024-12-08T08:15:04,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/c0eaea04c5314f8ba8e355f4d9a8aa94 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/c0eaea04c5314f8ba8e355f4d9a8aa94 2024-12-08T08:15:04,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/45a4fc15c8854d0882f187351dd3d38f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/45a4fc15c8854d0882f187351dd3d38f 2024-12-08T08:15:04,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/e087830bf8c94cc287e85052414b23f9 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/e087830bf8c94cc287e85052414b23f9 2024-12-08T08:15:04,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/061ff3a311be417195326c051d79b92b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/061ff3a311be417195326c051d79b92b 2024-12-08T08:15:04,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/19c6450281fc42e0b1f25a7e91dfb80e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/19c6450281fc42e0b1f25a7e91dfb80e 2024-12-08T08:15:04,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/5ba03976e0de47629553d44625bb9f91 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/5ba03976e0de47629553d44625bb9f91 2024-12-08T08:15:04,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/cdefbc56041340999764f22082a3337d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/cdefbc56041340999764f22082a3337d 2024-12-08T08:15:04,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/4eab6745862a485e92111bf74644c910 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/4eab6745862a485e92111bf74644c910 2024-12-08T08:15:04,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/ee045b44503e44b1b89e41d8f3c2cf27 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/ee045b44503e44b1b89e41d8f3c2cf27 2024-12-08T08:15:04,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/cfe442c60b6e4d6db3f1eb815e9d6188 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/cfe442c60b6e4d6db3f1eb815e9d6188 2024-12-08T08:15:04,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/8d357626d1a341e796844fe03e53ed45 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/8d357626d1a341e796844fe03e53ed45 2024-12-08T08:15:04,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/83889a7ef56949f7bbb395dea554bcdb to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/83889a7ef56949f7bbb395dea554bcdb 2024-12-08T08:15:04,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/c3ee9b417cbf4837a7aac4008ae5c989 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/c3ee9b417cbf4837a7aac4008ae5c989 2024-12-08T08:15:04,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/9e7e9095fd7d4d3eaff895a06f08dc91 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/9e7e9095fd7d4d3eaff895a06f08dc91 2024-12-08T08:15:04,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/55b3dd36b33545c7bcf7c6858f7f2195 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/55b3dd36b33545c7bcf7c6858f7f2195 2024-12-08T08:15:04,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/1436c99689164124b8607a746d46fcd4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/1436c99689164124b8607a746d46fcd4 2024-12-08T08:15:04,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/15b335183f254d3ca028739bcee950b9 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/15b335183f254d3ca028739bcee950b9 2024-12-08T08:15:04,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/f3420548a3dd449e88aef839489e7069 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/f3420548a3dd449e88aef839489e7069 2024-12-08T08:15:04,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/4cd58941a5ab47b1a46fa48e4d2bbac8 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/4cd58941a5ab47b1a46fa48e4d2bbac8 2024-12-08T08:15:04,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/f87cb50391a04079be8d70ea62f70e76 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/f87cb50391a04079be8d70ea62f70e76 2024-12-08T08:15:04,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/ce69fd0573e543958def59d405576300 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/ce69fd0573e543958def59d405576300 2024-12-08T08:15:04,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/a42d24620d5345358ba6594e38347e30 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/a42d24620d5345358ba6594e38347e30 2024-12-08T08:15:04,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/861b424c480847e6a2a170187feae019 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/861b424c480847e6a2a170187feae019 2024-12-08T08:15:04,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T08:15:04,943 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/recovered.edits/436.seqid, newMaxSeqId=436, maxSeqId=4 2024-12-08T08:15:04,944 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. 2024-12-08T08:15:04,944 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 45f279431c7ebbce8a61010c6b97623a: Waiting for close lock at 1733645703968Running coprocessor pre-close hooks at 1733645703968Disabling compacts and flushes for region at 1733645703968Disabling writes for close at 1733645703968Obtaining lock to block concurrent updates at 1733645703968Preparing flush snapshotting stores in 45f279431c7ebbce8a61010c6b97623a at 1733645703968Finished memstore snapshotting TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733645703969 (+1 ms)Flushing stores of TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a. at 1733645703969Flushing 45f279431c7ebbce8a61010c6b97623a/A: creating writer at 1733645703969Flushing mob file 45f279431c7ebbce8a61010c6b97623a/A: appending metadata at 1733645703975 (+6 ms)Flushing mob file 45f279431c7ebbce8a61010c6b97623a/A: closing flushed file at 1733645703975Flushing 45f279431c7ebbce8a61010c6b97623a/A: appending metadata at 1733645703989 (+14 ms)Flushing 45f279431c7ebbce8a61010c6b97623a/A: closing flushed file at 1733645703989Flushing 45f279431c7ebbce8a61010c6b97623a/B: creating writer at 1733645704000 (+11 ms)Flushing 45f279431c7ebbce8a61010c6b97623a/B: appending metadata at 1733645704003 (+3 ms)Flushing 45f279431c7ebbce8a61010c6b97623a/B: closing flushed file at 1733645704003Flushing 45f279431c7ebbce8a61010c6b97623a/C: creating writer at 1733645704424 (+421 ms)Flushing 45f279431c7ebbce8a61010c6b97623a/C: appending metadata at 1733645704427 (+3 ms)Flushing 45f279431c7ebbce8a61010c6b97623a/C: closing flushed file at 1733645704427Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ba3d1e4: reopening flushed file at 1733645704835 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45d1ff7d: reopening flushed file at 1733645704840 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f80bee2: reopening flushed file at 1733645704845 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 45f279431c7ebbce8a61010c6b97623a in 882ms, sequenceid=433, compaction requested=false at 1733645704850 (+5 ms)Writing region close event to WAL at 1733645704940 (+90 ms)Running coprocessor post-close hooks at 1733645704943 (+3 ms)Closed at 1733645704944 (+1 ms) 2024-12-08T08:15:04,946 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:04,946 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=45f279431c7ebbce8a61010c6b97623a, regionState=CLOSED 2024-12-08T08:15:04,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:15:04,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-08T08:15:04,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 45f279431c7ebbce8a61010c6b97623a, server=422e41b75614,38619,1733645641981 in 1.1330 sec 2024-12-08T08:15:04,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-08T08:15:04,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f279431c7ebbce8a61010c6b97623a, UNASSIGN in 1.1390 sec 2024-12-08T08:15:04,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-08T08:15:04,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1420 sec 2024-12-08T08:15:04,954 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645704954"}]},"ts":"1733645704954"} 2024-12-08T08:15:04,955 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T08:15:04,955 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T08:15:04,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-12-08T08:15:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T08:15:05,951 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:05,951 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-08T08:15:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:05,953 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-08T08:15:05,954 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:05,956 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,958 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/recovered.edits] 2024-12-08T08:15:05,961 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/358f51b5930640299c6bd285945a8fb3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/358f51b5930640299c6bd285945a8fb3 2024-12-08T08:15:05,962 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/f710f47fefe144e58068d70a601946f7 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/A/f710f47fefe144e58068d70a601946f7 2024-12-08T08:15:05,964 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/9ba1c1d750cb4514a837f864686e4525 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/9ba1c1d750cb4514a837f864686e4525 2024-12-08T08:15:05,965 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e12d6a4206c743599f90faf15d7f29bb to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/B/e12d6a4206c743599f90faf15d7f29bb 2024-12-08T08:15:05,969 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/13c41f28b0954629b44a3b7fcfe87dfb to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/13c41f28b0954629b44a3b7fcfe87dfb 2024-12-08T08:15:05,970 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/33d97e14695d417ab81c1482ad71fde4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/C/33d97e14695d417ab81c1482ad71fde4 2024-12-08T08:15:05,973 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/recovered.edits/436.seqid to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a/recovered.edits/436.seqid 2024-12-08T08:15:05,974 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,974 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T08:15:05,975 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T08:15:05,976 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T08:15:05,980 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120809b1537397ad4f08a084ca2f06609a04_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120809b1537397ad4f08a084ca2f06609a04_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,982 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080b5e32e0d4794803ae7c05c8426275af_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080b5e32e0d4794803ae7c05c8426275af_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,983 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208155be74d2d604e2ea5a60554d6d5c40a_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208155be74d2d604e2ea5a60554d6d5c40a_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,984 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208395eb77371374f9eafb74bb77e841963_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208395eb77371374f9eafb74bb77e841963_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,985 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084fdac473dd6f4d539e49ecca66b502a2_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084fdac473dd6f4d539e49ecca66b502a2_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,986 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120853a94eda9fa2464a8598370b05b854a4_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120853a94eda9fa2464a8598370b05b854a4_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,987 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085b254c4742d5421ba25eaead16fb250a_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085b254c4742d5421ba25eaead16fb250a_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,989 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086e0e3c97dcb44fb8b2cbc5f921e3d32b_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086e0e3c97dcb44fb8b2cbc5f921e3d32b_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,990 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120885283bb08a38461e9826c718c4027632_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120885283bb08a38461e9826c718c4027632_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,991 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120888252ff0a2404da08db085b94103033c_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120888252ff0a2404da08db085b94103033c_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,992 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089c6a741e7e56493e99960e4e2cc3d1e3_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089c6a741e7e56493e99960e4e2cc3d1e3_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,994 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208aa0e58770ef742bb86089522474d6156_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208aa0e58770ef742bb86089522474d6156_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,995 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b7740cc0e32146279a4237c43a2661c5_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b7740cc0e32146279a4237c43a2661c5_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,996 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b7ca789b76bf4fba9a6f7807d7af6e75_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b7ca789b76bf4fba9a6f7807d7af6e75_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,997 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c010e2e1a2554ca68e84277bce3c6071_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c010e2e1a2554ca68e84277bce3c6071_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,998 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c3db1f80ef834615ac8ea3df6662d27a_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c3db1f80ef834615ac8ea3df6662d27a_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:05,999 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d18b0d21c35b46529c9541183c30dce4_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d18b0d21c35b46529c9541183c30dce4_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:06,000 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d42a866fb28f41cdaaebc7332d1ba3b6_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d42a866fb28f41cdaaebc7332d1ba3b6_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:06,001 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208da32fc2b54b049539fa6fc2c4020af86_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208da32fc2b54b049539fa6fc2c4020af86_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:06,002 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ef6caca2eba1482187eb68b98d89dc2f_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ef6caca2eba1482187eb68b98d89dc2f_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:06,004 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f0d35d005f46478b9b9a68c263bf5321_45f279431c7ebbce8a61010c6b97623a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f0d35d005f46478b9b9a68c263bf5321_45f279431c7ebbce8a61010c6b97623a 2024-12-08T08:15:06,004 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T08:15:06,006 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:06,009 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T08:15:06,011 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T08:15:06,013 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:06,013 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T08:15:06,013 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733645706013"}]},"ts":"9223372036854775807"} 2024-12-08T08:15:06,015 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T08:15:06,015 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 45f279431c7ebbce8a61010c6b97623a, NAME => 'TestAcidGuarantees,,1733645672123.45f279431c7ebbce8a61010c6b97623a.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T08:15:06,016 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T08:15:06,016 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733645706016"}]},"ts":"9223372036854775807"} 2024-12-08T08:15:06,018 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T08:15:06,019 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:06,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-12-08T08:15:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-08T08:15:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T08:15:06,061 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:06,070 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 239) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2010449533_22 at /127.0.0.1:51048 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/cluster_7f2d8a74-a01c-b592-2655-65641f2d4b95/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/cluster_7f2d8a74-a01c-b592-2655-65641f2d4b95/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=386 (was 373) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7460 (was 6439) - AvailableMemoryMB LEAK? - 2024-12-08T08:15:06,077 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=386, ProcessCount=11, AvailableMemoryMB=7460 2024-12-08T08:15:06,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T08:15:06,079 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:15:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:06,081 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:15:06,081 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:06,081 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-12-08T08:15:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-08T08:15:06,082 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:15:06,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742061_1237 (size=1061) 2024-12-08T08:15:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-08T08:15:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-08T08:15:06,490 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b 2024-12-08T08:15:06,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742062_1238 (size=53) 2024-12-08T08:15:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-08T08:15:06,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:15:06,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f652a2fb0fbb79b2d0f57e1ac321a224, disabling compactions & flushes 2024-12-08T08:15:06,906 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:06,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:06,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. after waiting 0 ms 2024-12-08T08:15:06,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:06,906 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:06,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f652a2fb0fbb79b2d0f57e1ac321a224: Waiting for close lock at 1733645706906Disabling compacts and flushes for region at 1733645706906Disabling writes for close at 1733645706906Writing region close event to WAL at 1733645706906Closed at 1733645706906 2024-12-08T08:15:06,908 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:15:06,908 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733645706908"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733645706908"}]},"ts":"1733645706908"} 2024-12-08T08:15:06,909 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:15:06,910 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:15:06,911 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645706910"}]},"ts":"1733645706910"} 2024-12-08T08:15:06,912 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T08:15:06,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f652a2fb0fbb79b2d0f57e1ac321a224, ASSIGN}] 2024-12-08T08:15:06,913 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f652a2fb0fbb79b2d0f57e1ac321a224, ASSIGN 2024-12-08T08:15:06,914 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f652a2fb0fbb79b2d0f57e1ac321a224, ASSIGN; state=OFFLINE, location=422e41b75614,38619,1733645641981; forceNewPlan=false, retain=false 2024-12-08T08:15:07,065 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=f652a2fb0fbb79b2d0f57e1ac321a224, regionState=OPENING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:15:07,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f652a2fb0fbb79b2d0f57e1ac321a224, ASSIGN because future has completed 2024-12-08T08:15:07,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:15:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-08T08:15:07,223 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:07,223 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:15:07,224 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:07,224 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:15:07,224 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:07,224 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:07,226 INFO [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:07,228 INFO [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:15:07,228 INFO [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f652a2fb0fbb79b2d0f57e1ac321a224 columnFamilyName A 2024-12-08T08:15:07,229 DEBUG [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:07,230 INFO [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] regionserver.HStore(327): Store=f652a2fb0fbb79b2d0f57e1ac321a224/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:15:07,230 INFO [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:07,232 INFO [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:15:07,232 INFO [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f652a2fb0fbb79b2d0f57e1ac321a224 columnFamilyName B 2024-12-08T08:15:07,233 DEBUG [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:07,233 INFO [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] regionserver.HStore(327): Store=f652a2fb0fbb79b2d0f57e1ac321a224/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:15:07,233 INFO [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:07,234 INFO [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:15:07,235 INFO [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f652a2fb0fbb79b2d0f57e1ac321a224 columnFamilyName C 2024-12-08T08:15:07,235 DEBUG [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:07,235 INFO [StoreOpener-f652a2fb0fbb79b2d0f57e1ac321a224-1 {}] regionserver.HStore(327): Store=f652a2fb0fbb79b2d0f57e1ac321a224/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:15:07,235 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:07,235 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:07,236 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:07,237 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:07,238 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:07,238 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:07,238 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T08:15:07,239 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:07,241 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:15:07,241 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened f652a2fb0fbb79b2d0f57e1ac321a224; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60059344, jitterRate=-0.10504603385925293}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T08:15:07,241 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:07,242 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for f652a2fb0fbb79b2d0f57e1ac321a224: Running coprocessor pre-open hook at 1733645707224Writing region info on filesystem at 1733645707224Initializing all the Stores at 1733645707225 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645707225Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645707226 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645707226Cleaning up temporary data from old regions at 1733645707238 (+12 ms)Running coprocessor post-open hooks at 1733645707241 (+3 ms)Region opened successfully at 1733645707242 (+1 ms) 2024-12-08T08:15:07,243 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., pid=63, masterSystemTime=1733645707218 2024-12-08T08:15:07,244 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:07,244 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:07,245 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=f652a2fb0fbb79b2d0f57e1ac321a224, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:15:07,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:15:07,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-08T08:15:07,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 in 180 msec 2024-12-08T08:15:07,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-08T08:15:07,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f652a2fb0fbb79b2d0f57e1ac321a224, ASSIGN in 336 msec 2024-12-08T08:15:07,251 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:15:07,251 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645707251"}]},"ts":"1733645707251"} 2024-12-08T08:15:07,253 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T08:15:07,253 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:15:07,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-12-08T08:15:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-08T08:15:08,231 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:08,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12616921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:08,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:08,233 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c661a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:08,234 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5097cda6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,235 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:08,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:08,238 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T08:15:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 422e41b75614,44803,1733645641278 2024-12-08T08:15:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c15b409 2024-12-08T08:15:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T08:15:08,240 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T08:15:08,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c1be0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:08,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:08,241 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261e46b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:08,243 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3da23e68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:08,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:08,245 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733ef71f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:08,246 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c07228a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,247 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:08,247 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:08,248 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771a1a82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:08,249 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb8b0e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:08,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:08,251 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dbb50bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:08,253 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aec9ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,254 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:08,254 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:08,255 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f7200de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:08,256 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3603abc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:08,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:08,258 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24b21c3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:08,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d06d19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:08,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:08,261 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dcfed9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:08,263 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379a79b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:08,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:08,265 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65eb9587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:08,267 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b0495b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:08,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:08,269 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@742c620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:08,270 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@410cd236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:08,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:08,273 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5383b7b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:08,274 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@86c4428, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:08,276 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:08,276 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:08,277 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:08,277 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:08,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:08,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:08,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:08,280 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:08,280 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:15:08,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:15:08,282 DEBUG [Thread-1113 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:15:08,282 DEBUG [Thread-1113 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:08,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:08,284 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:15:08,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:15:08,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:15:08,295 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:08,296 DEBUG [Thread-1115 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:08,296 DEBUG [Thread-1115 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:08,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:08,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:15:08,300 DEBUG [Thread-1115 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:15:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:08,304 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:08,307 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:08,309 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:08,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:15:08,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:15:08,317 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-08T08:15:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T08:15:08,319 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:08,320 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:08,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:08,337 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:08,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:15:08,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733645708327, max timestamp=1733645708342Num uniques 859; 2024-12-08T08:15:08,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733645708327, max timestamp=1733645708342Num uniques 859; 2024-12-08T08:15:08,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733645708327, max timestamp=1733645708342Num uniques 859; 2024-12-08T08:15:08,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:15:08,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:15:08,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:15:08,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/3cae5893469d494db7999a77e0ae8084 is 50, key is test_row_0/A:col10/1733645708297/Put/seqid=0 2024-12-08T08:15:08,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645768383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645768386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645768387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645768389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645768389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742063_1239 (size=12001) 2024-12-08T08:15:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T08:15:08,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T08:15:08,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:08,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:08,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:08,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:08,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:08,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645768500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645768501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645768501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645768501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645768501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T08:15:08,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:08,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:08,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:08,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:08,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:08,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T08:15:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645768711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645768711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645768711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645768711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645768713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:08,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:08,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T08:15:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:08,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:08,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/3cae5893469d494db7999a77e0ae8084 2024-12-08T08:15:08,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/dec9feb069114e68a34847abb46093c3 is 50, key is test_row_0/B:col10/1733645708297/Put/seqid=0 2024-12-08T08:15:08,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742064_1240 (size=12001) 2024-12-08T08:15:08,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/dec9feb069114e68a34847abb46093c3 2024-12-08T08:15:08,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T08:15:08,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:08,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:08,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:08,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:08,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:08,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/919f3e602c904932b6aa26df5d79ca4c is 50, key is test_row_0/C:col10/1733645708297/Put/seqid=0 2024-12-08T08:15:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T08:15:08,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742065_1241 (size=12001) 2024-12-08T08:15:09,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645769022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645769022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645769022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645769023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645769023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T08:15:09,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:09,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:09,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:09,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:09,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:09,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T08:15:09,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:09,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:09,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:09,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:09,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:09,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/919f3e602c904932b6aa26df5d79ca4c 2024-12-08T08:15:09,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/3cae5893469d494db7999a77e0ae8084 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/3cae5893469d494db7999a77e0ae8084 2024-12-08T08:15:09,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T08:15:09,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:09,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:09,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:09,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:09,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:09,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/3cae5893469d494db7999a77e0ae8084, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T08:15:09,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/dec9feb069114e68a34847abb46093c3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/dec9feb069114e68a34847abb46093c3 2024-12-08T08:15:09,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/dec9feb069114e68a34847abb46093c3, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T08:15:09,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/919f3e602c904932b6aa26df5d79ca4c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/919f3e602c904932b6aa26df5d79ca4c 2024-12-08T08:15:09,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/919f3e602c904932b6aa26df5d79ca4c, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T08:15:09,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for f652a2fb0fbb79b2d0f57e1ac321a224 in 1118ms, sequenceid=12, compaction requested=false 2024-12-08T08:15:09,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T08:15:09,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-08T08:15:09,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:09,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:09,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:09,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:09,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:09,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:09,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/382e45f6c9d34a66acf0a374e99cb770 is 50, key is test_row_0/A:col10/1733645709535/Put/seqid=0 2024-12-08T08:15:09,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T08:15:09,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:09,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645769542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:09,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:09,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645769549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645769550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645769550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645769552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742066_1242 (size=12001) 2024-12-08T08:15:09,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/382e45f6c9d34a66acf0a374e99cb770 2024-12-08T08:15:09,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/8617170716d44237b88d4c66e79e3244 is 50, key is test_row_0/B:col10/1733645709535/Put/seqid=0 2024-12-08T08:15:09,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742067_1243 (size=12001) 2024-12-08T08:15:09,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/8617170716d44237b88d4c66e79e3244 2024-12-08T08:15:09,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/5072fb5b8e7b4bb5b305f2584c6cf6cb is 50, key is test_row_0/C:col10/1733645709535/Put/seqid=0 2024-12-08T08:15:09,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742068_1244 (size=12001) 2024-12-08T08:15:09,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/5072fb5b8e7b4bb5b305f2584c6cf6cb 2024-12-08T08:15:09,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645769661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/382e45f6c9d34a66acf0a374e99cb770 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/382e45f6c9d34a66acf0a374e99cb770 2024-12-08T08:15:09,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645769661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645769661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645769661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/382e45f6c9d34a66acf0a374e99cb770, entries=150, sequenceid=49, filesize=11.7 K 2024-12-08T08:15:09,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/8617170716d44237b88d4c66e79e3244 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/8617170716d44237b88d4c66e79e3244 2024-12-08T08:15:09,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645769670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/8617170716d44237b88d4c66e79e3244, entries=150, sequenceid=49, filesize=11.7 K 2024-12-08T08:15:09,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/5072fb5b8e7b4bb5b305f2584c6cf6cb as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/5072fb5b8e7b4bb5b305f2584c6cf6cb 2024-12-08T08:15:09,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/5072fb5b8e7b4bb5b305f2584c6cf6cb, entries=150, sequenceid=49, filesize=11.7 K 2024-12-08T08:15:09,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=46.96 KB/48090 for f652a2fb0fbb79b2d0f57e1ac321a224 in 146ms, sequenceid=49, compaction requested=false 2024-12-08T08:15:09,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:09,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T08:15:09,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:09,705 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:15:09,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:09,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:09,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:09,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:09,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:09,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:09,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/f0eb9bde7c50423aacfd1d90c7dd9202 is 50, key is test_row_0/A:col10/1733645709546/Put/seqid=0 2024-12-08T08:15:09,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742069_1245 (size=12001) 2024-12-08T08:15:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:09,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:09,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645769890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645709889, max timestamp=1733645709894Num uniques 859; 2024-12-08T08:15:09,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:15:09,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645709889, max timestamp=1733645709894Num uniques 859; 2024-12-08T08:15:09,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:15:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645709889, max timestamp=1733645709894Num uniques 859; 2024-12-08T08:15:09,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:15:09,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645769924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645769926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645769927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645769928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645770010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645770041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645770041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645770041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645770041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,121 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/f0eb9bde7c50423aacfd1d90c7dd9202 2024-12-08T08:15:10,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/75db671f249d4f24876100253a15f4e5 is 50, key is test_row_0/B:col10/1733645709546/Put/seqid=0 2024-12-08T08:15:10,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742070_1246 (size=12001) 2024-12-08T08:15:10,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645770220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645770251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645770251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645770251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645770251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T08:15:10,493 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T08:15:10,532 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/75db671f249d4f24876100253a15f4e5 2024-12-08T08:15:10,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645770530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/3b681692de5f4f9d930152b734d3a93e is 50, key is test_row_0/C:col10/1733645709546/Put/seqid=0 2024-12-08T08:15:10,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742071_1247 (size=12001) 2024-12-08T08:15:10,559 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/3b681692de5f4f9d930152b734d3a93e 2024-12-08T08:15:10,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645770560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645770561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645770561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645770561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:10,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/f0eb9bde7c50423aacfd1d90c7dd9202 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/f0eb9bde7c50423aacfd1d90c7dd9202 2024-12-08T08:15:10,570 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/f0eb9bde7c50423aacfd1d90c7dd9202, entries=150, sequenceid=59, filesize=11.7 K 2024-12-08T08:15:10,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/75db671f249d4f24876100253a15f4e5 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/75db671f249d4f24876100253a15f4e5 2024-12-08T08:15:10,581 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/75db671f249d4f24876100253a15f4e5, entries=150, sequenceid=59, filesize=11.7 K 2024-12-08T08:15:10,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/3b681692de5f4f9d930152b734d3a93e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/3b681692de5f4f9d930152b734d3a93e 2024-12-08T08:15:10,589 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/3b681692de5f4f9d930152b734d3a93e, entries=150, sequenceid=59, filesize=11.7 K 2024-12-08T08:15:10,590 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f652a2fb0fbb79b2d0f57e1ac321a224 in 885ms, sequenceid=59, compaction requested=true 2024-12-08T08:15:10,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:10,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:10,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-08T08:15:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-08T08:15:10,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-08T08:15:10,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2710 sec 2024-12-08T08:15:10,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.2770 sec 2024-12-08T08:15:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:11,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-08T08:15:11,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:11,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T08:15:11,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:11,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T08:15:11,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:11,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T08:15:11,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/4b2e82a70900455dad65caef4d089b3a is 50, key is test_row_0/A:col10/1733645709924/Put/seqid=0 2024-12-08T08:15:11,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742072_1248 (size=14341) 2024-12-08T08:15:11,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/4b2e82a70900455dad65caef4d089b3a 2024-12-08T08:15:11,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/eebbc795ecb24ff1874786cfe794e3c6 is 50, key is test_row_0/B:col10/1733645709924/Put/seqid=0 2024-12-08T08:15:11,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742073_1249 (size=12001) 2024-12-08T08:15:11,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645771074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645771075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645771077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645771077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645771077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645771190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645771190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645771191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645771191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645771191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645771400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645771401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645771401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645771401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645771401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/eebbc795ecb24ff1874786cfe794e3c6 2024-12-08T08:15:11,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/2dfe3f8fc60c49ecbf64a1305e3933bc is 50, key is test_row_0/C:col10/1733645709924/Put/seqid=0 2024-12-08T08:15:11,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742074_1250 (size=12001) 2024-12-08T08:15:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645771710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645771711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645771711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645771711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645771711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:11,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/2dfe3f8fc60c49ecbf64a1305e3933bc 2024-12-08T08:15:11,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/4b2e82a70900455dad65caef4d089b3a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4b2e82a70900455dad65caef4d089b3a 2024-12-08T08:15:11,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4b2e82a70900455dad65caef4d089b3a, entries=200, sequenceid=97, filesize=14.0 K 2024-12-08T08:15:11,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/eebbc795ecb24ff1874786cfe794e3c6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/eebbc795ecb24ff1874786cfe794e3c6 2024-12-08T08:15:11,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/eebbc795ecb24ff1874786cfe794e3c6, entries=150, sequenceid=97, filesize=11.7 K 2024-12-08T08:15:11,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/2dfe3f8fc60c49ecbf64a1305e3933bc as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2dfe3f8fc60c49ecbf64a1305e3933bc 2024-12-08T08:15:11,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2dfe3f8fc60c49ecbf64a1305e3933bc, entries=150, sequenceid=97, filesize=11.7 K 2024-12-08T08:15:11,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=53.67 KB/54960 for f652a2fb0fbb79b2d0f57e1ac321a224 in 872ms, sequenceid=97, compaction requested=true 2024-12-08T08:15:11,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:11,925 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:15:11,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:11,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:11,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:11,926 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:15:11,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:11,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:11,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:11,927 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:15:11,927 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/A is initiating minor compaction (all files) 2024-12-08T08:15:11,927 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:15:11,927 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/A in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:11,927 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/B is initiating minor compaction (all files) 2024-12-08T08:15:11,927 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/B in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:11,927 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/3cae5893469d494db7999a77e0ae8084, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/382e45f6c9d34a66acf0a374e99cb770, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/f0eb9bde7c50423aacfd1d90c7dd9202, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4b2e82a70900455dad65caef4d089b3a] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=49.2 K 2024-12-08T08:15:11,927 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/dec9feb069114e68a34847abb46093c3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/8617170716d44237b88d4c66e79e3244, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/75db671f249d4f24876100253a15f4e5, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/eebbc795ecb24ff1874786cfe794e3c6] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=46.9 K 2024-12-08T08:15:11,928 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cae5893469d494db7999a77e0ae8084, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733645708292 2024-12-08T08:15:11,928 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting dec9feb069114e68a34847abb46093c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733645708292 2024-12-08T08:15:11,928 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 8617170716d44237b88d4c66e79e3244, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733645708384 2024-12-08T08:15:11,928 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 382e45f6c9d34a66acf0a374e99cb770, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733645708384 2024-12-08T08:15:11,928 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 75db671f249d4f24876100253a15f4e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733645709546 2024-12-08T08:15:11,928 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0eb9bde7c50423aacfd1d90c7dd9202, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733645709546 2024-12-08T08:15:11,929 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting eebbc795ecb24ff1874786cfe794e3c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733645709924 2024-12-08T08:15:11,929 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b2e82a70900455dad65caef4d089b3a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733645709923 2024-12-08T08:15:11,940 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#B#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:11,941 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/e25d4481317a4f178530187e47731c38 is 50, key is test_row_0/B:col10/1733645709924/Put/seqid=0 2024-12-08T08:15:11,948 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#A#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:11,949 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/084df6063e924bb19d9b227c3865d216 is 50, key is test_row_0/A:col10/1733645709924/Put/seqid=0 2024-12-08T08:15:11,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742075_1251 (size=12139) 2024-12-08T08:15:11,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742076_1252 (size=12139) 2024-12-08T08:15:11,975 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/e25d4481317a4f178530187e47731c38 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/e25d4481317a4f178530187e47731c38 2024-12-08T08:15:11,978 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/084df6063e924bb19d9b227c3865d216 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/084df6063e924bb19d9b227c3865d216 2024-12-08T08:15:11,990 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/B of f652a2fb0fbb79b2d0f57e1ac321a224 into e25d4481317a4f178530187e47731c38(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:11,990 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:11,990 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/B, priority=12, startTime=1733645711925; duration=0sec 2024-12-08T08:15:11,990 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:11,990 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:B 2024-12-08T08:15:11,990 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:15:11,993 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/A of f652a2fb0fbb79b2d0f57e1ac321a224 into 084df6063e924bb19d9b227c3865d216(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:11,993 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:11,993 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/A, priority=12, startTime=1733645711925; duration=0sec 2024-12-08T08:15:11,993 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:11,993 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:A 2024-12-08T08:15:11,992 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:15:11,993 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/C is initiating minor compaction (all files) 2024-12-08T08:15:11,993 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/C in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:11,993 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/919f3e602c904932b6aa26df5d79ca4c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/5072fb5b8e7b4bb5b305f2584c6cf6cb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/3b681692de5f4f9d930152b734d3a93e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2dfe3f8fc60c49ecbf64a1305e3933bc] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=46.9 K 2024-12-08T08:15:11,994 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 919f3e602c904932b6aa26df5d79ca4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733645708292 2024-12-08T08:15:11,994 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 5072fb5b8e7b4bb5b305f2584c6cf6cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733645708384 2024-12-08T08:15:11,994 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b681692de5f4f9d930152b734d3a93e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733645709546 2024-12-08T08:15:11,995 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dfe3f8fc60c49ecbf64a1305e3933bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733645709924 2024-12-08T08:15:12,015 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#C#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:12,015 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/e8e423bca62f447ca872b4bb11e90aca is 50, key is test_row_0/C:col10/1733645709924/Put/seqid=0 2024-12-08T08:15:12,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742077_1253 (size=12139) 2024-12-08T08:15:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:12,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T08:15:12,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:12,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:12,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:12,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:12,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:12,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:12,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/76064382fb45402ba5755651c9418ae4 is 50, key is test_row_0/A:col10/1733645712222/Put/seqid=0 2024-12-08T08:15:12,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742078_1254 (size=9657) 2024-12-08T08:15:12,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/76064382fb45402ba5755651c9418ae4 2024-12-08T08:15:12,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/549d47fa81b34542b86847326b50f454 is 50, key is test_row_0/B:col10/1733645712222/Put/seqid=0 2024-12-08T08:15:12,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742079_1255 (size=9657) 2024-12-08T08:15:12,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/549d47fa81b34542b86847326b50f454 2024-12-08T08:15:12,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645772239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645772244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645772244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/8cadfc6d601d40c4a88b864eb7fedbcd is 50, key is test_row_0/C:col10/1733645712222/Put/seqid=0 2024-12-08T08:15:12,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645772245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645772245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742080_1256 (size=9657) 2024-12-08T08:15:12,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645772350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645772350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645772350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645772361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645772361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,427 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/e8e423bca62f447ca872b4bb11e90aca as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/e8e423bca62f447ca872b4bb11e90aca 2024-12-08T08:15:12,434 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/C of f652a2fb0fbb79b2d0f57e1ac321a224 into e8e423bca62f447ca872b4bb11e90aca(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:12,434 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:12,434 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/C, priority=12, startTime=1733645711926; duration=0sec 2024-12-08T08:15:12,434 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:12,434 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:C 2024-12-08T08:15:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T08:15:12,481 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:12,481 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-08T08:15:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T08:15:12,483 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:12,484 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:12,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:12,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645772560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645772560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645772561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645772571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645772571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T08:15:12,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T08:15:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:12,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:12,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:12,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/8cadfc6d601d40c4a88b864eb7fedbcd 2024-12-08T08:15:12,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/76064382fb45402ba5755651c9418ae4 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/76064382fb45402ba5755651c9418ae4 2024-12-08T08:15:12,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/76064382fb45402ba5755651c9418ae4, entries=100, sequenceid=112, filesize=9.4 K 2024-12-08T08:15:12,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/549d47fa81b34542b86847326b50f454 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/549d47fa81b34542b86847326b50f454 2024-12-08T08:15:12,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/549d47fa81b34542b86847326b50f454, entries=100, sequenceid=112, filesize=9.4 K 2024-12-08T08:15:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/8cadfc6d601d40c4a88b864eb7fedbcd as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8cadfc6d601d40c4a88b864eb7fedbcd 2024-12-08T08:15:12,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8cadfc6d601d40c4a88b864eb7fedbcd, entries=100, sequenceid=112, filesize=9.4 K 2024-12-08T08:15:12,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f652a2fb0fbb79b2d0f57e1ac321a224 in 449ms, sequenceid=112, compaction requested=false 2024-12-08T08:15:12,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:12,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T08:15:12,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:12,789 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T08:15:12,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:12,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:12,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:12,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:12,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:12,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:12,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/1d4e7d950f21442696102474447e359c is 50, key is test_row_0/A:col10/1733645712243/Put/seqid=0 2024-12-08T08:15:12,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742081_1257 (size=12101) 2024-12-08T08:15:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T08:15:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:12,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:12,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645772881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645772881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645772882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645772882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645772885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645772991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645772991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645772991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645772991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:13,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T08:15:13,201 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/1d4e7d950f21442696102474447e359c 2024-12-08T08:15:13,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645773201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645773201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645773201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:13,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645773205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:13,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/80e6e723c5dd4ddb83a2880806e4d1b0 is 50, key is test_row_0/B:col10/1733645712243/Put/seqid=0 2024-12-08T08:15:13,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742082_1258 (size=12101) 2024-12-08T08:15:13,235 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/80e6e723c5dd4ddb83a2880806e4d1b0 2024-12-08T08:15:13,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/ff98330c2c0a417990d0b02935c941c6 is 50, key is test_row_0/C:col10/1733645712243/Put/seqid=0 2024-12-08T08:15:13,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742083_1259 (size=12101) 2024-12-08T08:15:13,267 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/ff98330c2c0a417990d0b02935c941c6 2024-12-08T08:15:13,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/1d4e7d950f21442696102474447e359c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1d4e7d950f21442696102474447e359c 2024-12-08T08:15:13,277 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1d4e7d950f21442696102474447e359c, entries=150, sequenceid=132, filesize=11.8 K 2024-12-08T08:15:13,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/80e6e723c5dd4ddb83a2880806e4d1b0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/80e6e723c5dd4ddb83a2880806e4d1b0 2024-12-08T08:15:13,285 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/80e6e723c5dd4ddb83a2880806e4d1b0, entries=150, sequenceid=132, filesize=11.8 K 2024-12-08T08:15:13,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/ff98330c2c0a417990d0b02935c941c6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/ff98330c2c0a417990d0b02935c941c6 2024-12-08T08:15:13,292 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/ff98330c2c0a417990d0b02935c941c6, entries=150, sequenceid=132, filesize=11.8 K 2024-12-08T08:15:13,293 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f652a2fb0fbb79b2d0f57e1ac321a224 in 503ms, sequenceid=132, compaction requested=true 2024-12-08T08:15:13,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:13,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:13,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-08T08:15:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-08T08:15:13,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-08T08:15:13,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 810 msec 2024-12-08T08:15:13,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 817 msec 2024-12-08T08:15:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:13,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T08:15:13,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:13,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:13,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:13,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:13,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:13,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:13,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/4561df2f2c4946568952b7690e9bb069 is 50, key is test_row_0/A:col10/1733645712881/Put/seqid=0 2024-12-08T08:15:13,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742084_1260 (size=14541) 2024-12-08T08:15:13,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645773493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:13,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645773510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:13,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645773510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645773511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645773511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:13,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:13,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645773601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T08:15:13,621 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:13,621 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-08T08:15:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T08:15:13,623 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:13,623 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:13,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T08:15:13,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T08:15:13,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:13,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:13,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:13,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:13,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:13,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:13,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645773811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:13,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/4561df2f2c4946568952b7690e9bb069 2024-12-08T08:15:13,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/9c71fe68c53a429086c75aa9768f023c is 50, key is test_row_0/B:col10/1733645712881/Put/seqid=0 2024-12-08T08:15:13,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742085_1261 (size=12151) 2024-12-08T08:15:13,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/9c71fe68c53a429086c75aa9768f023c 2024-12-08T08:15:13,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/2a6ca539455344bf8836fcfd79f83f0c is 50, key is test_row_0/C:col10/1733645712881/Put/seqid=0 2024-12-08T08:15:13,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742086_1262 (size=12151) 2024-12-08T08:15:13,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/2a6ca539455344bf8836fcfd79f83f0c 2024-12-08T08:15:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/4561df2f2c4946568952b7690e9bb069 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4561df2f2c4946568952b7690e9bb069 2024-12-08T08:15:13,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4561df2f2c4946568952b7690e9bb069, entries=200, sequenceid=146, filesize=14.2 K 2024-12-08T08:15:13,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/9c71fe68c53a429086c75aa9768f023c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/9c71fe68c53a429086c75aa9768f023c 2024-12-08T08:15:13,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/9c71fe68c53a429086c75aa9768f023c, entries=150, sequenceid=146, filesize=11.9 K 2024-12-08T08:15:13,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/2a6ca539455344bf8836fcfd79f83f0c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2a6ca539455344bf8836fcfd79f83f0c 2024-12-08T08:15:13,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2a6ca539455344bf8836fcfd79f83f0c, entries=150, sequenceid=146, filesize=11.9 K 2024-12-08T08:15:13,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f652a2fb0fbb79b2d0f57e1ac321a224 in 519ms, sequenceid=146, compaction requested=true 2024-12-08T08:15:13,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:13,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:13,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:13,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:13,923 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:15:13,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:13,923 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:15:13,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:13,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:13,925 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:15:13,925 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48438 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:15:13,925 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/B is initiating minor compaction (all files) 2024-12-08T08:15:13,925 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/A is initiating minor compaction (all files) 2024-12-08T08:15:13,925 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/B in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:13,925 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/A in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:13,926 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/084df6063e924bb19d9b227c3865d216, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/76064382fb45402ba5755651c9418ae4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1d4e7d950f21442696102474447e359c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4561df2f2c4946568952b7690e9bb069] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=47.3 K 2024-12-08T08:15:13,926 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/e25d4481317a4f178530187e47731c38, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/549d47fa81b34542b86847326b50f454, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/80e6e723c5dd4ddb83a2880806e4d1b0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/9c71fe68c53a429086c75aa9768f023c] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=45.0 K 2024-12-08T08:15:13,926 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 084df6063e924bb19d9b227c3865d216, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733645709924 2024-12-08T08:15:13,926 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting e25d4481317a4f178530187e47731c38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733645709924 2024-12-08T08:15:13,927 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76064382fb45402ba5755651c9418ae4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733645712221 2024-12-08T08:15:13,928 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 549d47fa81b34542b86847326b50f454, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733645712221 2024-12-08T08:15:13,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T08:15:13,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:13,929 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T08:15:13,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:13,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:13,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:13,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:13,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:13,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:13,933 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 80e6e723c5dd4ddb83a2880806e4d1b0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733645712241 2024-12-08T08:15:13,933 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d4e7d950f21442696102474447e359c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733645712241 2024-12-08T08:15:13,933 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c71fe68c53a429086c75aa9768f023c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733645712881 2024-12-08T08:15:13,934 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4561df2f2c4946568952b7690e9bb069, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733645712881 2024-12-08T08:15:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/1502157682034aaf949e250316bbe8c8 is 50, key is test_row_0/A:col10/1733645713460/Put/seqid=0 2024-12-08T08:15:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T08:15:13,967 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#B#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:13,967 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#A#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:13,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742087_1263 (size=12151) 2024-12-08T08:15:13,967 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/93d9491fd6a44648870baad7c2df7414 is 50, key is test_row_0/B:col10/1733645712881/Put/seqid=0 2024-12-08T08:15:13,967 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/08b779602dcc4b5593bfdff8b81c1243 is 50, key is test_row_0/A:col10/1733645712881/Put/seqid=0 2024-12-08T08:15:13,973 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/1502157682034aaf949e250316bbe8c8 2024-12-08T08:15:13,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/b768595a582b4b13b0d487169fc80738 is 50, key is test_row_0/B:col10/1733645713460/Put/seqid=0 2024-12-08T08:15:14,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742088_1264 (size=12425) 2024-12-08T08:15:14,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742089_1265 (size=12425) 2024-12-08T08:15:14,020 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/08b779602dcc4b5593bfdff8b81c1243 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/08b779602dcc4b5593bfdff8b81c1243 2024-12-08T08:15:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:14,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:14,025 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/A of f652a2fb0fbb79b2d0f57e1ac321a224 into 08b779602dcc4b5593bfdff8b81c1243(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:14,025 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:14,025 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/A, priority=12, startTime=1733645713923; duration=0sec 2024-12-08T08:15:14,026 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:14,026 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:A 2024-12-08T08:15:14,026 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:15:14,032 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:15:14,032 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/C is initiating minor compaction (all files) 2024-12-08T08:15:14,033 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/C in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:14,033 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/e8e423bca62f447ca872b4bb11e90aca, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8cadfc6d601d40c4a88b864eb7fedbcd, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/ff98330c2c0a417990d0b02935c941c6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2a6ca539455344bf8836fcfd79f83f0c] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=45.0 K 2024-12-08T08:15:14,033 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8e423bca62f447ca872b4bb11e90aca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733645709924 2024-12-08T08:15:14,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742090_1266 (size=12151) 2024-12-08T08:15:14,034 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cadfc6d601d40c4a88b864eb7fedbcd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733645712221 2024-12-08T08:15:14,035 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff98330c2c0a417990d0b02935c941c6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733645712241 2024-12-08T08:15:14,035 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/b768595a582b4b13b0d487169fc80738 2024-12-08T08:15:14,036 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a6ca539455344bf8836fcfd79f83f0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733645712881 2024-12-08T08:15:14,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645774035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/815be12c3cbf4b53a75d042482feb328 is 50, key is test_row_0/C:col10/1733645713460/Put/seqid=0 2024-12-08T08:15:14,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645774036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645774042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645774042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,067 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:14,067 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/6efaaf08210d483ebc63875fceed7fbf is 50, key is test_row_0/C:col10/1733645712881/Put/seqid=0 2024-12-08T08:15:14,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742091_1267 (size=12151) 2024-12-08T08:15:14,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645774120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742092_1268 (size=12425) 2024-12-08T08:15:14,140 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/6efaaf08210d483ebc63875fceed7fbf as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/6efaaf08210d483ebc63875fceed7fbf 2024-12-08T08:15:14,145 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/C of f652a2fb0fbb79b2d0f57e1ac321a224 into 6efaaf08210d483ebc63875fceed7fbf(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:14,145 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:14,145 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/C, priority=12, startTime=1733645713923; duration=0sec 2024-12-08T08:15:14,145 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:14,145 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:C 2024-12-08T08:15:14,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645774150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645774150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645774161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645774161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T08:15:14,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645774361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645774361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645774370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645774370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,413 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/93d9491fd6a44648870baad7c2df7414 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/93d9491fd6a44648870baad7c2df7414 2024-12-08T08:15:14,419 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/B of f652a2fb0fbb79b2d0f57e1ac321a224 into 93d9491fd6a44648870baad7c2df7414(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:14,419 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:14,419 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/B, priority=12, startTime=1733645713923; duration=0sec 2024-12-08T08:15:14,419 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:14,419 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:B 2024-12-08T08:15:14,504 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/815be12c3cbf4b53a75d042482feb328 2024-12-08T08:15:14,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/1502157682034aaf949e250316bbe8c8 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1502157682034aaf949e250316bbe8c8 2024-12-08T08:15:14,517 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1502157682034aaf949e250316bbe8c8, entries=150, sequenceid=164, filesize=11.9 K 2024-12-08T08:15:14,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/b768595a582b4b13b0d487169fc80738 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/b768595a582b4b13b0d487169fc80738 2024-12-08T08:15:14,524 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/b768595a582b4b13b0d487169fc80738, entries=150, sequenceid=164, filesize=11.9 K 2024-12-08T08:15:14,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/815be12c3cbf4b53a75d042482feb328 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/815be12c3cbf4b53a75d042482feb328 2024-12-08T08:15:14,531 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/815be12c3cbf4b53a75d042482feb328, entries=150, sequenceid=164, filesize=11.9 K 2024-12-08T08:15:14,533 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f652a2fb0fbb79b2d0f57e1ac321a224 in 603ms, sequenceid=164, compaction requested=false 2024-12-08T08:15:14,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:14,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:14,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-08T08:15:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-08T08:15:14,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-08T08:15:14,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 911 msec 2024-12-08T08:15:14,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 920 msec 2024-12-08T08:15:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:14,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T08:15:14,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:14,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:14,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:14,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:14,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:14,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:14,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/07974b0d77754db68900b908fc999133 is 50, key is test_row_0/A:col10/1733645714040/Put/seqid=0 2024-12-08T08:15:14,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742093_1269 (size=14541) 2024-12-08T08:15:14,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645774672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645774674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645774677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645774682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645774682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T08:15:14,761 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:14,761 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-08T08:15:14,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T08:15:14,762 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:14,764 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:14,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:14,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645774781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645774790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645774790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T08:15:14,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T08:15:14,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:14,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:14,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:14,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:14,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:14,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645774991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:15,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645775000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645775000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:15,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/07974b0d77754db68900b908fc999133 2024-12-08T08:15:15,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/a665a17c6aa44e128035de77476f3eff is 50, key is test_row_0/B:col10/1733645714040/Put/seqid=0 2024-12-08T08:15:15,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742094_1270 (size=12151) 2024-12-08T08:15:15,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T08:15:15,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:15,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:15,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:15,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:15,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T08:15:15,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645775191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:15,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645775191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:15,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:15,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T08:15:15,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:15,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:15,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:15,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:15,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:15,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645775301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:15,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645775310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:15,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645775310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:15,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T08:15:15,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:15,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:15,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:15,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:15,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T08:15:15,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/a665a17c6aa44e128035de77476f3eff 2024-12-08T08:15:15,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/e637d38476594cae81e06d33d1aa1cf3 is 50, key is test_row_0/C:col10/1733645714040/Put/seqid=0 2024-12-08T08:15:15,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742095_1271 (size=12151) 2024-12-08T08:15:15,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/e637d38476594cae81e06d33d1aa1cf3 2024-12-08T08:15:15,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/07974b0d77754db68900b908fc999133 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/07974b0d77754db68900b908fc999133 2024-12-08T08:15:15,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/07974b0d77754db68900b908fc999133, entries=200, sequenceid=182, filesize=14.2 K 2024-12-08T08:15:15,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/a665a17c6aa44e128035de77476f3eff as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a665a17c6aa44e128035de77476f3eff 2024-12-08T08:15:15,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a665a17c6aa44e128035de77476f3eff, entries=150, sequenceid=182, filesize=11.9 K 2024-12-08T08:15:15,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/e637d38476594cae81e06d33d1aa1cf3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/e637d38476594cae81e06d33d1aa1cf3 2024-12-08T08:15:15,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/e637d38476594cae81e06d33d1aa1cf3, entries=150, sequenceid=182, filesize=11.9 K 2024-12-08T08:15:15,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f652a2fb0fbb79b2d0f57e1ac321a224 in 864ms, sequenceid=182, compaction requested=true 2024-12-08T08:15:15,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:15,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:15,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:15,496 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:15,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:15,496 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:15,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:15,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:15,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:15,497 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:15,497 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:15,497 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/A is initiating minor compaction (all files) 2024-12-08T08:15:15,497 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/B is initiating minor compaction (all files) 2024-12-08T08:15:15,497 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/B in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:15,497 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/A in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:15,497 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/08b779602dcc4b5593bfdff8b81c1243, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1502157682034aaf949e250316bbe8c8, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/07974b0d77754db68900b908fc999133] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=38.2 K 2024-12-08T08:15:15,497 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/93d9491fd6a44648870baad7c2df7414, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/b768595a582b4b13b0d487169fc80738, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a665a17c6aa44e128035de77476f3eff] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=35.9 K 2024-12-08T08:15:15,498 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08b779602dcc4b5593bfdff8b81c1243, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733645712881 2024-12-08T08:15:15,498 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 93d9491fd6a44648870baad7c2df7414, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733645712881 2024-12-08T08:15:15,498 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1502157682034aaf949e250316bbe8c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733645713460 2024-12-08T08:15:15,499 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting b768595a582b4b13b0d487169fc80738, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733645713460 2024-12-08T08:15:15,499 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07974b0d77754db68900b908fc999133, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733645714034 2024-12-08T08:15:15,499 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting a665a17c6aa44e128035de77476f3eff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733645714034 2024-12-08T08:15:15,506 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:15,506 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/a637347519054c968e6406c5bd2523c6 is 50, key is test_row_0/A:col10/1733645714040/Put/seqid=0 2024-12-08T08:15:15,523 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#B#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:15,525 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/8db62a11f5e24eb5a15e2f9d9c48c816 is 50, key is test_row_0/B:col10/1733645714040/Put/seqid=0 2024-12-08T08:15:15,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T08:15:15,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:15,527 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T08:15:15,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:15,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:15,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:15,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:15,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:15,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:15,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742096_1272 (size=12527) 2024-12-08T08:15:15,537 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/a637347519054c968e6406c5bd2523c6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/a637347519054c968e6406c5bd2523c6 2024-12-08T08:15:15,546 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/A of f652a2fb0fbb79b2d0f57e1ac321a224 into a637347519054c968e6406c5bd2523c6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:15,546 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:15,546 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/A, priority=13, startTime=1733645715496; duration=0sec 2024-12-08T08:15:15,546 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:15,546 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:A 2024-12-08T08:15:15,546 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:15,548 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:15,548 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/C is initiating minor compaction (all files) 2024-12-08T08:15:15,548 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/C in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:15,548 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/6efaaf08210d483ebc63875fceed7fbf, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/815be12c3cbf4b53a75d042482feb328, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/e637d38476594cae81e06d33d1aa1cf3] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=35.9 K 2024-12-08T08:15:15,548 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6efaaf08210d483ebc63875fceed7fbf, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733645712881 2024-12-08T08:15:15,549 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 815be12c3cbf4b53a75d042482feb328, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733645713460 2024-12-08T08:15:15,549 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e637d38476594cae81e06d33d1aa1cf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733645714034 2024-12-08T08:15:15,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742097_1273 (size=12527) 2024-12-08T08:15:15,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/c4dd69176de74a15b6b2de27c8f5e4dc is 50, key is test_row_0/A:col10/1733645714671/Put/seqid=0 2024-12-08T08:15:15,577 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/8db62a11f5e24eb5a15e2f9d9c48c816 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/8db62a11f5e24eb5a15e2f9d9c48c816 2024-12-08T08:15:15,584 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/B of f652a2fb0fbb79b2d0f57e1ac321a224 into 8db62a11f5e24eb5a15e2f9d9c48c816(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:15,584 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:15,584 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/B, priority=13, startTime=1733645715496; duration=0sec 2024-12-08T08:15:15,584 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:15,584 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:B 2024-12-08T08:15:15,586 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#C#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:15,586 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/2af4cd6e9d66408f89ad76e5d68e7d7c is 50, key is test_row_0/C:col10/1733645714040/Put/seqid=0 2024-12-08T08:15:15,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742098_1274 (size=12151) 2024-12-08T08:15:15,590 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/c4dd69176de74a15b6b2de27c8f5e4dc 2024-12-08T08:15:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/4fe104b7baea42d9b9950ffe95170988 is 50, key is test_row_0/B:col10/1733645714671/Put/seqid=0 2024-12-08T08:15:15,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742099_1275 (size=12527) 2024-12-08T08:15:15,628 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/2af4cd6e9d66408f89ad76e5d68e7d7c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2af4cd6e9d66408f89ad76e5d68e7d7c 2024-12-08T08:15:15,634 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/C of f652a2fb0fbb79b2d0f57e1ac321a224 into 2af4cd6e9d66408f89ad76e5d68e7d7c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:15,634 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:15,634 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/C, priority=13, startTime=1733645715496; duration=0sec 2024-12-08T08:15:15,634 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:15,635 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:C 2024-12-08T08:15:15,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742100_1276 (size=12151) 2024-12-08T08:15:15,643 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/4fe104b7baea42d9b9950ffe95170988 2024-12-08T08:15:15,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/0f24210e2c0a48ecb8c0998192894aaf is 50, key is test_row_0/C:col10/1733645714671/Put/seqid=0 2024-12-08T08:15:15,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742101_1277 (size=12151) 2024-12-08T08:15:15,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:15,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:15,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645775840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:15,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645775840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:15,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645775841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T08:15:15,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645775951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:15,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645775951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:16,083 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/0f24210e2c0a48ecb8c0998192894aaf 2024-12-08T08:15:16,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/c4dd69176de74a15b6b2de27c8f5e4dc as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/c4dd69176de74a15b6b2de27c8f5e4dc 2024-12-08T08:15:16,098 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/c4dd69176de74a15b6b2de27c8f5e4dc, entries=150, sequenceid=199, filesize=11.9 K 2024-12-08T08:15:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/4fe104b7baea42d9b9950ffe95170988 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/4fe104b7baea42d9b9950ffe95170988 2024-12-08T08:15:16,104 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/4fe104b7baea42d9b9950ffe95170988, entries=150, sequenceid=199, filesize=11.9 K 2024-12-08T08:15:16,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/0f24210e2c0a48ecb8c0998192894aaf as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/0f24210e2c0a48ecb8c0998192894aaf 2024-12-08T08:15:16,109 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/0f24210e2c0a48ecb8c0998192894aaf, entries=150, sequenceid=199, filesize=11.9 K 2024-12-08T08:15:16,111 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f652a2fb0fbb79b2d0f57e1ac321a224 in 584ms, sequenceid=199, compaction requested=false 2024-12-08T08:15:16,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:16,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:16,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-08T08:15:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-08T08:15:16,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-08T08:15:16,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3490 sec 2024-12-08T08:15:16,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.3550 sec 2024-12-08T08:15:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:16,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T08:15:16,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:16,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:16,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:16,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:16,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:16,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:16,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/6fa915853f3e4703a354fb15c82135b3 is 50, key is test_row_0/A:col10/1733645715840/Put/seqid=0 2024-12-08T08:15:16,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742102_1278 (size=16931) 2024-12-08T08:15:16,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645776185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:16,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645776187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:16,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645776211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:16,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645776211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:16,212 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in f652a2fb0fbb79b2d0f57e1ac321a224 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:16,213 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in f652a2fb0fbb79b2d0f57e1ac321a224 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:16,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645776291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:16,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645776300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:16,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645776500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:16,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645776511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:16,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/6fa915853f3e4703a354fb15c82135b3 2024-12-08T08:15:16,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/a48b2a9c27ac444583f4fcfe3312e8fb is 50, key is test_row_0/B:col10/1733645715840/Put/seqid=0 2024-12-08T08:15:16,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742103_1279 (size=12151) 2024-12-08T08:15:16,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645776810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:16,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:16,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645776820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:16,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645776851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:16,854 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_0' in f652a2fb0fbb79b2d0f57e1ac321a224 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T08:15:16,911 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:16,911 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-08T08:15:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T08:15:16,913 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:16,913 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:16,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:16,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/a48b2a9c27ac444583f4fcfe3312e8fb 2024-12-08T08:15:16,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/c351251b253b48cb8178b566672222e4 is 50, key is test_row_0/C:col10/1733645715840/Put/seqid=0 2024-12-08T08:15:16,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742104_1280 (size=12151) 2024-12-08T08:15:16,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/c351251b253b48cb8178b566672222e4 2024-12-08T08:15:17,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/6fa915853f3e4703a354fb15c82135b3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/6fa915853f3e4703a354fb15c82135b3 2024-12-08T08:15:17,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/6fa915853f3e4703a354fb15c82135b3, entries=250, sequenceid=219, filesize=16.5 K 2024-12-08T08:15:17,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/a48b2a9c27ac444583f4fcfe3312e8fb as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a48b2a9c27ac444583f4fcfe3312e8fb 2024-12-08T08:15:17,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a48b2a9c27ac444583f4fcfe3312e8fb, entries=150, sequenceid=219, filesize=11.9 K 2024-12-08T08:15:17,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/c351251b253b48cb8178b566672222e4 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/c351251b253b48cb8178b566672222e4 2024-12-08T08:15:17,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/c351251b253b48cb8178b566672222e4, entries=150, sequenceid=219, filesize=11.9 K 2024-12-08T08:15:17,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f652a2fb0fbb79b2d0f57e1ac321a224 in 853ms, sequenceid=219, compaction requested=true 2024-12-08T08:15:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:17,017 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:17,017 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:17,019 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:17,019 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/B is initiating minor compaction (all files) 2024-12-08T08:15:17,019 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/B in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:17,019 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/8db62a11f5e24eb5a15e2f9d9c48c816, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/4fe104b7baea42d9b9950ffe95170988, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a48b2a9c27ac444583f4fcfe3312e8fb] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.0 K 2024-12-08T08:15:17,019 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41609 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:17,019 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/A is initiating minor compaction (all files) 2024-12-08T08:15:17,019 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/A in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:17,019 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 8db62a11f5e24eb5a15e2f9d9c48c816, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733645714034 2024-12-08T08:15:17,019 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/a637347519054c968e6406c5bd2523c6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/c4dd69176de74a15b6b2de27c8f5e4dc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/6fa915853f3e4703a354fb15c82135b3] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=40.6 K 2024-12-08T08:15:17,020 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting a637347519054c968e6406c5bd2523c6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733645714034 2024-12-08T08:15:17,020 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fe104b7baea42d9b9950ffe95170988, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733645714669 2024-12-08T08:15:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T08:15:17,021 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4dd69176de74a15b6b2de27c8f5e4dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733645714669 2024-12-08T08:15:17,021 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting a48b2a9c27ac444583f4fcfe3312e8fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733645715839 2024-12-08T08:15:17,022 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fa915853f3e4703a354fb15c82135b3, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733645715828 2024-12-08T08:15:17,030 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:17,031 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/a0354cad0a964e79aa2274e022c444e2 is 50, key is test_row_0/B:col10/1733645715840/Put/seqid=0 2024-12-08T08:15:17,037 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#A#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:17,038 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/0e0ca28981754eedb28155749ac7b6e6 is 50, key is test_row_0/A:col10/1733645715840/Put/seqid=0 2024-12-08T08:15:17,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742105_1281 (size=12629) 2024-12-08T08:15:17,048 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/a0354cad0a964e79aa2274e022c444e2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a0354cad0a964e79aa2274e022c444e2 2024-12-08T08:15:17,055 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/B of f652a2fb0fbb79b2d0f57e1ac321a224 into a0354cad0a964e79aa2274e022c444e2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:17,055 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:17,055 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/B, priority=13, startTime=1733645717017; duration=0sec 2024-12-08T08:15:17,056 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:17,059 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:B 2024-12-08T08:15:17,059 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:17,061 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:17,061 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/C is initiating minor compaction (all files) 2024-12-08T08:15:17,061 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/C in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:17,062 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2af4cd6e9d66408f89ad76e5d68e7d7c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/0f24210e2c0a48ecb8c0998192894aaf, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/c351251b253b48cb8178b566672222e4] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.0 K 2024-12-08T08:15:17,062 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 2af4cd6e9d66408f89ad76e5d68e7d7c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733645714034 2024-12-08T08:15:17,062 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f24210e2c0a48ecb8c0998192894aaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733645714669 2024-12-08T08:15:17,063 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting c351251b253b48cb8178b566672222e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733645715839 2024-12-08T08:15:17,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T08:15:17,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:17,067 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T08:15:17,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:17,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:17,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:17,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:17,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:17,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:17,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/14f840ba5fcc41c391c0204789090430 is 50, key is test_row_0/A:col10/1733645716186/Put/seqid=0 2024-12-08T08:15:17,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742106_1282 (size=12629) 2024-12-08T08:15:17,079 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:17,079 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/a18434af0ef64d3e90e78738b87cd723 is 50, key is test_row_0/C:col10/1733645715840/Put/seqid=0 2024-12-08T08:15:17,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742108_1284 (size=12629) 2024-12-08T08:15:17,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742107_1283 (size=12151) 2024-12-08T08:15:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T08:15:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:17,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645777351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:17,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645777355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:17,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:17,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645777461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:17,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:17,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645777470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:17,483 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/0e0ca28981754eedb28155749ac7b6e6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/0e0ca28981754eedb28155749ac7b6e6 2024-12-08T08:15:17,493 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/A of f652a2fb0fbb79b2d0f57e1ac321a224 into 0e0ca28981754eedb28155749ac7b6e6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:17,493 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:17,493 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/A, priority=13, startTime=1733645717017; duration=0sec 2024-12-08T08:15:17,493 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:17,493 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:A 2024-12-08T08:15:17,499 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/14f840ba5fcc41c391c0204789090430 2024-12-08T08:15:17,505 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/a18434af0ef64d3e90e78738b87cd723 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/a18434af0ef64d3e90e78738b87cd723 2024-12-08T08:15:17,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/6a226dedd16d4a35adc47cbe76702e44 is 50, key is test_row_0/B:col10/1733645716186/Put/seqid=0 2024-12-08T08:15:17,513 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/C of f652a2fb0fbb79b2d0f57e1ac321a224 into a18434af0ef64d3e90e78738b87cd723(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:17,513 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:17,513 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/C, priority=13, startTime=1733645717017; duration=0sec 2024-12-08T08:15:17,513 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:17,513 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:C 2024-12-08T08:15:17,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742109_1285 (size=12151) 2024-12-08T08:15:17,530 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/6a226dedd16d4a35adc47cbe76702e44 2024-12-08T08:15:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/8af2687d91434725a4a07788d9fa1428 is 50, key is test_row_0/C:col10/1733645716186/Put/seqid=0 2024-12-08T08:15:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T08:15:17,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742110_1286 (size=12151) 2024-12-08T08:15:17,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645777671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:17,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645777680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:17,943 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/8af2687d91434725a4a07788d9fa1428 2024-12-08T08:15:17,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/14f840ba5fcc41c391c0204789090430 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/14f840ba5fcc41c391c0204789090430 2024-12-08T08:15:17,951 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/14f840ba5fcc41c391c0204789090430, entries=150, sequenceid=235, filesize=11.9 K 2024-12-08T08:15:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/6a226dedd16d4a35adc47cbe76702e44 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/6a226dedd16d4a35adc47cbe76702e44 2024-12-08T08:15:17,955 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/6a226dedd16d4a35adc47cbe76702e44, entries=150, sequenceid=235, filesize=11.9 K 2024-12-08T08:15:17,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/8af2687d91434725a4a07788d9fa1428 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8af2687d91434725a4a07788d9fa1428 2024-12-08T08:15:17,960 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8af2687d91434725a4a07788d9fa1428, entries=150, sequenceid=235, filesize=11.9 K 2024-12-08T08:15:17,961 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f652a2fb0fbb79b2d0f57e1ac321a224 in 894ms, sequenceid=235, compaction requested=false 2024-12-08T08:15:17,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:17,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:17,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-08T08:15:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-08T08:15:17,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-08T08:15:17,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0490 sec 2024-12-08T08:15:17,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0530 sec 2024-12-08T08:15:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:17,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T08:15:17,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:17,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:17,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:17,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:17,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:17,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:17,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/cbc77e2195dc443fb560f3fbc780fd87 is 50, key is test_row_0/A:col10/1733645717349/Put/seqid=0 2024-12-08T08:15:17,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742111_1287 (size=14541) 2024-12-08T08:15:18,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645778007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645778010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T08:15:18,050 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:18,051 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-08T08:15:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T08:15:18,052 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:18,053 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:18,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:18,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645778124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645778124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T08:15:18,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T08:15:18,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:18,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:18,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:18,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:18,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:18,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645778220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,223 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in f652a2fb0fbb79b2d0f57e1ac321a224 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645778241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,245 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in f652a2fb0fbb79b2d0f57e1ac321a224 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645778331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645778331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T08:15:18,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:18,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:18,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:18,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:18,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T08:15:18,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/cbc77e2195dc443fb560f3fbc780fd87 2024-12-08T08:15:18,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/c21e120fa7f349caab5a05e6800ae104 is 50, key is test_row_0/B:col10/1733645717349/Put/seqid=0 2024-12-08T08:15:18,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742112_1288 (size=12151) 2024-12-08T08:15:18,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/c21e120fa7f349caab5a05e6800ae104 2024-12-08T08:15:18,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/05c379a3abd247bea4bb497b20e6f580 is 50, key is test_row_0/C:col10/1733645717349/Put/seqid=0 2024-12-08T08:15:18,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742113_1289 (size=12151) 2024-12-08T08:15:18,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/05c379a3abd247bea4bb497b20e6f580 2024-12-08T08:15:18,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/cbc77e2195dc443fb560f3fbc780fd87 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/cbc77e2195dc443fb560f3fbc780fd87 2024-12-08T08:15:18,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/cbc77e2195dc443fb560f3fbc780fd87, entries=200, sequenceid=255, filesize=14.2 K 2024-12-08T08:15:18,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/c21e120fa7f349caab5a05e6800ae104 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c21e120fa7f349caab5a05e6800ae104 2024-12-08T08:15:18,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c21e120fa7f349caab5a05e6800ae104, entries=150, sequenceid=255, filesize=11.9 K 2024-12-08T08:15:18,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/05c379a3abd247bea4bb497b20e6f580 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/05c379a3abd247bea4bb497b20e6f580 2024-12-08T08:15:18,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/05c379a3abd247bea4bb497b20e6f580, entries=150, sequenceid=255, filesize=11.9 K 2024-12-08T08:15:18,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f652a2fb0fbb79b2d0f57e1ac321a224 in 450ms, sequenceid=255, compaction requested=true 2024-12-08T08:15:18,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:18,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:18,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:18,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:18,433 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:18,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:18,433 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:18,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:18,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:18,434 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:18,434 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:18,434 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/A is initiating minor compaction (all files) 2024-12-08T08:15:18,434 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/B is initiating minor compaction (all files) 2024-12-08T08:15:18,434 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/A in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:18,434 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/B in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:18,434 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/0e0ca28981754eedb28155749ac7b6e6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/14f840ba5fcc41c391c0204789090430, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/cbc77e2195dc443fb560f3fbc780fd87] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=38.4 K 2024-12-08T08:15:18,434 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a0354cad0a964e79aa2274e022c444e2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/6a226dedd16d4a35adc47cbe76702e44, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c21e120fa7f349caab5a05e6800ae104] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.1 K 2024-12-08T08:15:18,435 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting a0354cad0a964e79aa2274e022c444e2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733645715839 2024-12-08T08:15:18,435 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e0ca28981754eedb28155749ac7b6e6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733645715839 2024-12-08T08:15:18,435 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a226dedd16d4a35adc47cbe76702e44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733645716182 2024-12-08T08:15:18,435 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14f840ba5fcc41c391c0204789090430, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733645716182 2024-12-08T08:15:18,436 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting c21e120fa7f349caab5a05e6800ae104, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733645717345 2024-12-08T08:15:18,436 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc77e2195dc443fb560f3fbc780fd87, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733645717345 2024-12-08T08:15:18,442 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#B#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:18,443 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:18,443 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/f3d84f00738d47549dfe6e5c3d6dd879 is 50, key is test_row_0/B:col10/1733645717349/Put/seqid=0 2024-12-08T08:15:18,443 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/05cb7122e91b4f5b9a5ffb6c84af4a2b is 50, key is test_row_0/A:col10/1733645717349/Put/seqid=0 2024-12-08T08:15:18,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742114_1290 (size=12731) 2024-12-08T08:15:18,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742115_1291 (size=12731) 2024-12-08T08:15:18,453 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/05cb7122e91b4f5b9a5ffb6c84af4a2b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/05cb7122e91b4f5b9a5ffb6c84af4a2b 2024-12-08T08:15:18,458 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/A of f652a2fb0fbb79b2d0f57e1ac321a224 into 05cb7122e91b4f5b9a5ffb6c84af4a2b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:18,458 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:18,458 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/A, priority=13, startTime=1733645718433; duration=0sec 2024-12-08T08:15:18,458 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:18,458 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:A 2024-12-08T08:15:18,458 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:18,460 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:18,460 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/C is initiating minor compaction (all files) 2024-12-08T08:15:18,460 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/C in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:18,460 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/a18434af0ef64d3e90e78738b87cd723, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8af2687d91434725a4a07788d9fa1428, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/05c379a3abd247bea4bb497b20e6f580] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.1 K 2024-12-08T08:15:18,460 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting a18434af0ef64d3e90e78738b87cd723, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733645715839 2024-12-08T08:15:18,461 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8af2687d91434725a4a07788d9fa1428, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733645716182 2024-12-08T08:15:18,461 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05c379a3abd247bea4bb497b20e6f580, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733645717345 2024-12-08T08:15:18,476 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#C#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:18,477 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/aa24fea34bbd42e8b5ce861184a5572e is 50, key is test_row_0/C:col10/1733645717349/Put/seqid=0 2024-12-08T08:15:18,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742116_1292 (size=12731) 2024-12-08T08:15:18,498 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/aa24fea34bbd42e8b5ce861184a5572e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/aa24fea34bbd42e8b5ce861184a5572e 2024-12-08T08:15:18,508 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/C of f652a2fb0fbb79b2d0f57e1ac321a224 into aa24fea34bbd42e8b5ce861184a5572e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:18,508 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:18,508 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/C, priority=13, startTime=1733645718433; duration=0sec 2024-12-08T08:15:18,509 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:18,509 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:C 2024-12-08T08:15:18,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T08:15:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:18,511 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T08:15:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:18,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/4d38f3d5ac2848b98c59130f95c26060 is 50, key is test_row_0/A:col10/1733645718004/Put/seqid=0 2024-12-08T08:15:18,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742117_1293 (size=12301) 2024-12-08T08:15:18,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:18,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645778672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645778672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T08:15:18,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645778781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645778781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,851 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/f3d84f00738d47549dfe6e5c3d6dd879 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/f3d84f00738d47549dfe6e5c3d6dd879 2024-12-08T08:15:18,855 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/B of f652a2fb0fbb79b2d0f57e1ac321a224 into f3d84f00738d47549dfe6e5c3d6dd879(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:18,855 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:18,855 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/B, priority=13, startTime=1733645718433; duration=0sec 2024-12-08T08:15:18,855 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:18,855 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:B 2024-12-08T08:15:18,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645778871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,871 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_0' in f652a2fb0fbb79b2d0f57e1ac321a224 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,924 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/4d38f3d5ac2848b98c59130f95c26060 2024-12-08T08:15:18,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/250ee7918f5c4676ae1754e2260c4e9d is 50, key is test_row_0/B:col10/1733645718004/Put/seqid=0 2024-12-08T08:15:18,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742118_1294 (size=12301) 2024-12-08T08:15:18,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645778991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645778991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T08:15:19,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645779300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:19,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645779300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:19,340 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/250ee7918f5c4676ae1754e2260c4e9d 2024-12-08T08:15:19,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/48d267e0742542eebe5a1a33c2b01c41 is 50, key is test_row_0/C:col10/1733645718004/Put/seqid=0 2024-12-08T08:15:19,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742119_1295 (size=12301) 2024-12-08T08:15:19,752 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/48d267e0742542eebe5a1a33c2b01c41 2024-12-08T08:15:19,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/4d38f3d5ac2848b98c59130f95c26060 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4d38f3d5ac2848b98c59130f95c26060 2024-12-08T08:15:19,759 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4d38f3d5ac2848b98c59130f95c26060, entries=150, sequenceid=271, filesize=12.0 K 2024-12-08T08:15:19,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/250ee7918f5c4676ae1754e2260c4e9d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/250ee7918f5c4676ae1754e2260c4e9d 2024-12-08T08:15:19,763 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/250ee7918f5c4676ae1754e2260c4e9d, entries=150, sequenceid=271, filesize=12.0 K 2024-12-08T08:15:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/48d267e0742542eebe5a1a33c2b01c41 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/48d267e0742542eebe5a1a33c2b01c41 2024-12-08T08:15:19,769 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/48d267e0742542eebe5a1a33c2b01c41, entries=150, sequenceid=271, filesize=12.0 K 2024-12-08T08:15:19,769 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for f652a2fb0fbb79b2d0f57e1ac321a224 in 1258ms, sequenceid=271, compaction requested=false 2024-12-08T08:15:19,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:19,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:19,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-08T08:15:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-08T08:15:19,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-08T08:15:19,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7180 sec 2024-12-08T08:15:19,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.7220 sec 2024-12-08T08:15:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:19,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T08:15:19,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:19,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:19,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:19,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:19,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:19,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:19,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/e1789b4833c44019a7eef5aa80293ca2 is 50, key is test_row_0/A:col10/1733645718671/Put/seqid=0 2024-12-08T08:15:19,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645779827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:19,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645779830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:19,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:19,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:19,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:19,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742120_1296 (size=12301) 2024-12-08T08:15:19,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645779940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:19,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645779941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:20,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645780151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:20,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645780151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:20,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T08:15:20,201 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:20,201 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-08T08:15:20,203 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T08:15:20,203 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:20,203 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:20,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/e1789b4833c44019a7eef5aa80293ca2 2024-12-08T08:15:20,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/702743e73da0440387af700f6b926554 is 50, key is test_row_0/B:col10/1733645718671/Put/seqid=0 2024-12-08T08:15:20,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742121_1297 (size=12301) 2024-12-08T08:15:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T08:15:20,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T08:15:20,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:20,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:20,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:20,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645780460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:20,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645780460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:20,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T08:15:20,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:20,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:20,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:20,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T08:15:20,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/702743e73da0440387af700f6b926554 2024-12-08T08:15:20,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T08:15:20,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/48a458d49e834072addf30832961f457 is 50, key is test_row_0/C:col10/1733645718671/Put/seqid=0 2024-12-08T08:15:20,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:20,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:20,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:20,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742122_1298 (size=12301) 2024-12-08T08:15:20,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T08:15:20,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:20,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:20,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:20,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T08:15:20,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T08:15:20,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:20,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:20,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:20,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:20,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645780971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:20,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645780971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:21,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/48a458d49e834072addf30832961f457 2024-12-08T08:15:21,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/e1789b4833c44019a7eef5aa80293ca2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/e1789b4833c44019a7eef5aa80293ca2 2024-12-08T08:15:21,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/e1789b4833c44019a7eef5aa80293ca2, entries=150, sequenceid=292, filesize=12.0 K 2024-12-08T08:15:21,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/702743e73da0440387af700f6b926554 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/702743e73da0440387af700f6b926554 2024-12-08T08:15:21,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/702743e73da0440387af700f6b926554, entries=150, sequenceid=292, filesize=12.0 K 2024-12-08T08:15:21,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/48a458d49e834072addf30832961f457 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/48a458d49e834072addf30832961f457 2024-12-08T08:15:21,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/48a458d49e834072addf30832961f457, entries=150, sequenceid=292, filesize=12.0 K 2024-12-08T08:15:21,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f652a2fb0fbb79b2d0f57e1ac321a224 in 1278ms, sequenceid=292, compaction requested=true 2024-12-08T08:15:21,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:21,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:21,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:21,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:21,091 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:21,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:21,091 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:21,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:21,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:21,092 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:21,092 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:21,092 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/A is initiating minor compaction (all files) 2024-12-08T08:15:21,092 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/B is initiating minor compaction (all files) 2024-12-08T08:15:21,092 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/A in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:21,092 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/B in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:21,092 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/05cb7122e91b4f5b9a5ffb6c84af4a2b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4d38f3d5ac2848b98c59130f95c26060, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/e1789b4833c44019a7eef5aa80293ca2] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.5 K 2024-12-08T08:15:21,092 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/f3d84f00738d47549dfe6e5c3d6dd879, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/250ee7918f5c4676ae1754e2260c4e9d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/702743e73da0440387af700f6b926554] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.5 K 2024-12-08T08:15:21,092 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05cb7122e91b4f5b9a5ffb6c84af4a2b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733645717345 2024-12-08T08:15:21,092 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting f3d84f00738d47549dfe6e5c3d6dd879, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733645717345 2024-12-08T08:15:21,092 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d38f3d5ac2848b98c59130f95c26060, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733645718002 2024-12-08T08:15:21,093 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 250ee7918f5c4676ae1754e2260c4e9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733645718002 2024-12-08T08:15:21,093 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1789b4833c44019a7eef5aa80293ca2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733645718670 2024-12-08T08:15:21,093 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 702743e73da0440387af700f6b926554, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733645718670 2024-12-08T08:15:21,098 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:21,099 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/422dc58d870247b598dc257c65771403 is 50, key is test_row_0/B:col10/1733645718671/Put/seqid=0 2024-12-08T08:15:21,099 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#A#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:21,099 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/765229b428934e2495def6bd7ae86ea6 is 50, key is test_row_0/A:col10/1733645718671/Put/seqid=0 2024-12-08T08:15:21,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742123_1299 (size=12983) 2024-12-08T08:15:21,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742124_1300 (size=12983) 2024-12-08T08:15:21,111 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/422dc58d870247b598dc257c65771403 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/422dc58d870247b598dc257c65771403 2024-12-08T08:15:21,111 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/765229b428934e2495def6bd7ae86ea6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/765229b428934e2495def6bd7ae86ea6 2024-12-08T08:15:21,116 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/B of f652a2fb0fbb79b2d0f57e1ac321a224 into 422dc58d870247b598dc257c65771403(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:21,116 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:21,116 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/B, priority=13, startTime=1733645721091; duration=0sec 2024-12-08T08:15:21,116 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/A of f652a2fb0fbb79b2d0f57e1ac321a224 into 765229b428934e2495def6bd7ae86ea6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:21,116 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:21,116 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:21,116 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/A, priority=13, startTime=1733645721091; duration=0sec 2024-12-08T08:15:21,116 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:B 2024-12-08T08:15:21,116 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:21,116 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:21,116 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:A 2024-12-08T08:15:21,118 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:21,118 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/C is initiating minor compaction (all files) 2024-12-08T08:15:21,118 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/C in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:21,118 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/aa24fea34bbd42e8b5ce861184a5572e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/48d267e0742542eebe5a1a33c2b01c41, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/48a458d49e834072addf30832961f457] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.5 K 2024-12-08T08:15:21,118 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting aa24fea34bbd42e8b5ce861184a5572e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733645717345 2024-12-08T08:15:21,119 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 48d267e0742542eebe5a1a33c2b01c41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733645718002 2024-12-08T08:15:21,119 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 48a458d49e834072addf30832961f457, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733645718670 2024-12-08T08:15:21,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T08:15:21,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:21,124 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T08:15:21,124 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#C#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:21,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:21,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:21,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:21,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:21,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:21,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:21,125 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/ea8535702ae14594a6d1e2c9a461b642 is 50, key is test_row_0/C:col10/1733645718671/Put/seqid=0 2024-12-08T08:15:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/185e7a03b6d54767acd6d585d72047cf is 50, key is test_row_0/A:col10/1733645719828/Put/seqid=0 2024-12-08T08:15:21,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742125_1301 (size=12983) 2024-12-08T08:15:21,151 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/ea8535702ae14594a6d1e2c9a461b642 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/ea8535702ae14594a6d1e2c9a461b642 2024-12-08T08:15:21,155 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/C of f652a2fb0fbb79b2d0f57e1ac321a224 into ea8535702ae14594a6d1e2c9a461b642(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:21,155 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:21,156 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/C, priority=13, startTime=1733645721091; duration=0sec 2024-12-08T08:15:21,156 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:21,156 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:C 2024-12-08T08:15:21,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742126_1302 (size=12301) 2024-12-08T08:15:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T08:15:21,559 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/185e7a03b6d54767acd6d585d72047cf 2024-12-08T08:15:21,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/c99f1023569b4ba18125ba8600aeb654 is 50, key is test_row_0/B:col10/1733645719828/Put/seqid=0 2024-12-08T08:15:21,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742127_1303 (size=12301) 2024-12-08T08:15:21,969 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/c99f1023569b4ba18125ba8600aeb654 2024-12-08T08:15:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:21,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:21,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/063730bde9e64fbbb31b10bc3d6a8200 is 50, key is test_row_0/C:col10/1733645719828/Put/seqid=0 2024-12-08T08:15:22,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742128_1304 (size=12301) 2024-12-08T08:15:22,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645782032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:22,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645782035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:22,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645782141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645782141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:22,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733645782231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:22,233 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in f652a2fb0fbb79b2d0f57e1ac321a224 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:22,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733645782281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:22,284 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in f652a2fb0fbb79b2d0f57e1ac321a224 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T08:15:22,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645782351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:22,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645782352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:22,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:22,413 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/063730bde9e64fbbb31b10bc3d6a8200 2024-12-08T08:15:22,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/185e7a03b6d54767acd6d585d72047cf as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/185e7a03b6d54767acd6d585d72047cf 2024-12-08T08:15:22,430 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/185e7a03b6d54767acd6d585d72047cf, entries=150, sequenceid=306, filesize=12.0 K 2024-12-08T08:15:22,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/c99f1023569b4ba18125ba8600aeb654 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c99f1023569b4ba18125ba8600aeb654 2024-12-08T08:15:22,441 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c99f1023569b4ba18125ba8600aeb654, entries=150, sequenceid=306, filesize=12.0 K 2024-12-08T08:15:22,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/063730bde9e64fbbb31b10bc3d6a8200 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/063730bde9e64fbbb31b10bc3d6a8200 2024-12-08T08:15:22,459 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/063730bde9e64fbbb31b10bc3d6a8200, entries=150, sequenceid=306, filesize=12.0 K 2024-12-08T08:15:22,461 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f652a2fb0fbb79b2d0f57e1ac321a224 in 1336ms, sequenceid=306, compaction requested=false 2024-12-08T08:15:22,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:22,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:22,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-08T08:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-08T08:15:22,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-08T08:15:22,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2600 sec 2024-12-08T08:15:22,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.2660 sec 2024-12-08T08:15:22,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645722022, max timestamp=1733645722663Num uniques 859; 2024-12-08T08:15:22,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:15:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:22,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=96.16 KB heapSize=282.03 KB 2024-12-08T08:15:22,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:22,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:22,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:22,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:22,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:22,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:22,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/b7069d83703648ce814e5933f2e14353 is 50, key is test_row_0/A:col10/1733645722664/Put/seqid=0 2024-12-08T08:15:22,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645782712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:22,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:22,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645782713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:22,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:22,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742129_1305 (size=12301) 2024-12-08T08:15:22,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/b7069d83703648ce814e5933f2e14353 2024-12-08T08:15:22,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/20ea1320dedd4be6b4f8eda2cf01075e is 50, key is test_row_0/B:col10/1733645722664/Put/seqid=0 2024-12-08T08:15:22,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742130_1306 (size=12301) 2024-12-08T08:15:22,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/20ea1320dedd4be6b4f8eda2cf01075e 2024-12-08T08:15:22,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/d530ba7569df40ddbaacd77bfbb61b6d is 50, key is test_row_0/C:col10/1733645722664/Put/seqid=0 2024-12-08T08:15:22,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742131_1307 (size=12301) 2024-12-08T08:15:22,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/d530ba7569df40ddbaacd77bfbb61b6d 2024-12-08T08:15:22,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/b7069d83703648ce814e5933f2e14353 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b7069d83703648ce814e5933f2e14353 2024-12-08T08:15:22,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645782831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:22,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645782831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:22,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b7069d83703648ce814e5933f2e14353, entries=150, sequenceid=329, filesize=12.0 K 2024-12-08T08:15:22,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/20ea1320dedd4be6b4f8eda2cf01075e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/20ea1320dedd4be6b4f8eda2cf01075e 2024-12-08T08:15:22,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/20ea1320dedd4be6b4f8eda2cf01075e, entries=150, sequenceid=329, filesize=12.0 K 2024-12-08T08:15:22,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/d530ba7569df40ddbaacd77bfbb61b6d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/d530ba7569df40ddbaacd77bfbb61b6d 2024-12-08T08:15:22,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/d530ba7569df40ddbaacd77bfbb61b6d, entries=150, sequenceid=329, filesize=12.0 K 2024-12-08T08:15:22,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=87.22 KB/89310 for f652a2fb0fbb79b2d0f57e1ac321a224 in 200ms, sequenceid=329, compaction requested=true 2024-12-08T08:15:22,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:22,868 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:22,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:22,869 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:22,869 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/A is initiating minor compaction (all files) 2024-12-08T08:15:22,869 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/A in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:22,869 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/765229b428934e2495def6bd7ae86ea6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/185e7a03b6d54767acd6d585d72047cf, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b7069d83703648ce814e5933f2e14353] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.7 K 2024-12-08T08:15:22,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:22,870 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:22,870 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 765229b428934e2495def6bd7ae86ea6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733645718670 2024-12-08T08:15:22,871 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 185e7a03b6d54767acd6d585d72047cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733645719823 2024-12-08T08:15:22,871 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:22,871 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/B is initiating minor compaction (all files) 2024-12-08T08:15:22,871 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/B in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:22,871 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/422dc58d870247b598dc257c65771403, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c99f1023569b4ba18125ba8600aeb654, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/20ea1320dedd4be6b4f8eda2cf01075e] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.7 K 2024-12-08T08:15:22,872 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 422dc58d870247b598dc257c65771403, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733645718670 2024-12-08T08:15:22,872 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7069d83703648ce814e5933f2e14353, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733645722029 2024-12-08T08:15:22,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:22,873 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting c99f1023569b4ba18125ba8600aeb654, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733645719823 2024-12-08T08:15:22,873 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 20ea1320dedd4be6b4f8eda2cf01075e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733645722029 2024-12-08T08:15:22,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:22,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:22,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:22,895 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#A#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:22,896 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/fdc5ed286c7e40d38efd5de8983d5b0e is 50, key is test_row_0/A:col10/1733645722664/Put/seqid=0 2024-12-08T08:15:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:22,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T08:15:22,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:22,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:22,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:22,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:22,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:22,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:22,905 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#B#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:22,905 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/c77b2225ee81499aaa3f24af4df7deda is 50, key is test_row_0/B:col10/1733645722664/Put/seqid=0 2024-12-08T08:15:22,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/9f25274e04404722b0d4ade9ba24d256 is 50, key is test_row_0/A:col10/1733645722902/Put/seqid=0 2024-12-08T08:15:22,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742132_1308 (size=13085) 2024-12-08T08:15:22,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742133_1309 (size=13085) 2024-12-08T08:15:22,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742134_1310 (size=12301) 2024-12-08T08:15:22,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645782988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:23,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645783040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:23,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645783040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:23,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645783101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:23,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645783310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:23,339 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/fdc5ed286c7e40d38efd5de8983d5b0e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/fdc5ed286c7e40d38efd5de8983d5b0e 2024-12-08T08:15:23,347 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/A of f652a2fb0fbb79b2d0f57e1ac321a224 into fdc5ed286c7e40d38efd5de8983d5b0e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:23,347 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:23,348 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/A, priority=13, startTime=1733645722868; duration=0sec 2024-12-08T08:15:23,348 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:23,348 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:A 2024-12-08T08:15:23,348 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:23,349 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:23,349 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/C is initiating minor compaction (all files) 2024-12-08T08:15:23,349 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/C in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:23,349 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/ea8535702ae14594a6d1e2c9a461b642, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/063730bde9e64fbbb31b10bc3d6a8200, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/d530ba7569df40ddbaacd77bfbb61b6d] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.7 K 2024-12-08T08:15:23,350 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea8535702ae14594a6d1e2c9a461b642, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733645718670 2024-12-08T08:15:23,350 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 063730bde9e64fbbb31b10bc3d6a8200, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733645719823 2024-12-08T08:15:23,350 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting d530ba7569df40ddbaacd77bfbb61b6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733645722029 2024-12-08T08:15:23,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645783351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:23,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645783351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:23,360 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#C#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:23,360 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/787e492105c642a7a6fc60ec0a60a9a3 is 50, key is test_row_0/C:col10/1733645722664/Put/seqid=0 2024-12-08T08:15:23,364 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/c77b2225ee81499aaa3f24af4df7deda as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c77b2225ee81499aaa3f24af4df7deda 2024-12-08T08:15:23,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/9f25274e04404722b0d4ade9ba24d256 2024-12-08T08:15:23,378 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/B of f652a2fb0fbb79b2d0f57e1ac321a224 into c77b2225ee81499aaa3f24af4df7deda(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:23,378 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:23,378 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/B, priority=13, startTime=1733645722870; duration=0sec 2024-12-08T08:15:23,378 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:23,378 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:B 2024-12-08T08:15:23,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/90c174741e514a40a3a8241626de6197 is 50, key is test_row_0/B:col10/1733645722902/Put/seqid=0 2024-12-08T08:15:23,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742135_1311 (size=13085) 2024-12-08T08:15:23,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742136_1312 (size=12301) 2024-12-08T08:15:23,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/90c174741e514a40a3a8241626de6197 2024-12-08T08:15:23,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/6a632a9fca244c2f8e47648a7b492f62 is 50, key is test_row_0/C:col10/1733645722902/Put/seqid=0 2024-12-08T08:15:23,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742137_1313 (size=12301) 2024-12-08T08:15:23,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645783621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:23,787 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/787e492105c642a7a6fc60ec0a60a9a3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/787e492105c642a7a6fc60ec0a60a9a3 2024-12-08T08:15:23,792 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/C of f652a2fb0fbb79b2d0f57e1ac321a224 into 787e492105c642a7a6fc60ec0a60a9a3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:23,792 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:23,792 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/C, priority=13, startTime=1733645722875; duration=0sec 2024-12-08T08:15:23,792 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:23,792 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:C 2024-12-08T08:15:23,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/6a632a9fca244c2f8e47648a7b492f62 2024-12-08T08:15:23,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/9f25274e04404722b0d4ade9ba24d256 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/9f25274e04404722b0d4ade9ba24d256 2024-12-08T08:15:23,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/9f25274e04404722b0d4ade9ba24d256, entries=150, sequenceid=346, filesize=12.0 K 2024-12-08T08:15:23,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/90c174741e514a40a3a8241626de6197 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/90c174741e514a40a3a8241626de6197 2024-12-08T08:15:23,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/90c174741e514a40a3a8241626de6197, entries=150, sequenceid=346, filesize=12.0 K 2024-12-08T08:15:23,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/6a632a9fca244c2f8e47648a7b492f62 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/6a632a9fca244c2f8e47648a7b492f62 2024-12-08T08:15:23,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/6a632a9fca244c2f8e47648a7b492f62, entries=150, sequenceid=346, filesize=12.0 K 2024-12-08T08:15:23,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f652a2fb0fbb79b2d0f57e1ac321a224 in 936ms, sequenceid=346, compaction requested=false 2024-12-08T08:15:23,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:23,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T08:15:23,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:23,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:23,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:23,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:23,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:23,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:23,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/8032295376b146bd99c3cab5c104a4ca is 50, key is test_row_0/A:col10/1733645722986/Put/seqid=0 2024-12-08T08:15:23,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645783900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:23,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645783902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:23,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742138_1314 (size=12301) 2024-12-08T08:15:24,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645784011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:24,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645784011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:24,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645784130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:24,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:24,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645784221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:24,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645784221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:24,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/8032295376b146bd99c3cab5c104a4ca 2024-12-08T08:15:24,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/829bbaa975d34101bdf9c61c509aaa0b is 50, key is test_row_0/B:col10/1733645722986/Put/seqid=0 2024-12-08T08:15:24,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742139_1315 (size=12301) 2024-12-08T08:15:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T08:15:24,371 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:24,371 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-08T08:15:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T08:15:24,374 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:24,375 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:24,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T08:15:24,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T08:15:24,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:24,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:24,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:24,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:24,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:24,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645784530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:24,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645784530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:24,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T08:15:24,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:24,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:24,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:24,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:24,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T08:15:24,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/829bbaa975d34101bdf9c61c509aaa0b 2024-12-08T08:15:24,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/10cda65b971e42e9b51e38a0692a2ad0 is 50, key is test_row_0/C:col10/1733645722986/Put/seqid=0 2024-12-08T08:15:24,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742140_1316 (size=12301) 2024-12-08T08:15:24,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/10cda65b971e42e9b51e38a0692a2ad0 2024-12-08T08:15:24,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/8032295376b146bd99c3cab5c104a4ca as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/8032295376b146bd99c3cab5c104a4ca 2024-12-08T08:15:24,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/8032295376b146bd99c3cab5c104a4ca, entries=150, sequenceid=366, filesize=12.0 K 2024-12-08T08:15:24,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/829bbaa975d34101bdf9c61c509aaa0b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/829bbaa975d34101bdf9c61c509aaa0b 2024-12-08T08:15:24,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/829bbaa975d34101bdf9c61c509aaa0b, entries=150, sequenceid=366, filesize=12.0 K 2024-12-08T08:15:24,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/10cda65b971e42e9b51e38a0692a2ad0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/10cda65b971e42e9b51e38a0692a2ad0 2024-12-08T08:15:24,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/10cda65b971e42e9b51e38a0692a2ad0, entries=150, sequenceid=366, filesize=12.0 K 2024-12-08T08:15:24,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f652a2fb0fbb79b2d0f57e1ac321a224 in 947ms, sequenceid=366, compaction requested=true 2024-12-08T08:15:24,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:24,810 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:24,811 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:24,813 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:24,813 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/B is initiating minor compaction (all files) 2024-12-08T08:15:24,813 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/B in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:24,813 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c77b2225ee81499aaa3f24af4df7deda, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/90c174741e514a40a3a8241626de6197, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/829bbaa975d34101bdf9c61c509aaa0b] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.8 K 2024-12-08T08:15:24,813 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:24,813 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/A is initiating minor compaction (all files) 2024-12-08T08:15:24,814 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/A in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:24,814 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting c77b2225ee81499aaa3f24af4df7deda, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733645722029 2024-12-08T08:15:24,814 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/fdc5ed286c7e40d38efd5de8983d5b0e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/9f25274e04404722b0d4ade9ba24d256, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/8032295376b146bd99c3cab5c104a4ca] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.8 K 2024-12-08T08:15:24,814 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 90c174741e514a40a3a8241626de6197, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733645722699 2024-12-08T08:15:24,814 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdc5ed286c7e40d38efd5de8983d5b0e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733645722029 2024-12-08T08:15:24,814 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 829bbaa975d34101bdf9c61c509aaa0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733645722976 2024-12-08T08:15:24,814 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f25274e04404722b0d4ade9ba24d256, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733645722699 2024-12-08T08:15:24,815 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8032295376b146bd99c3cab5c104a4ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733645722976 2024-12-08T08:15:24,824 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#A#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:24,824 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/295c04bc0b834be9a127088cfae9b2d1 is 50, key is test_row_0/A:col10/1733645722986/Put/seqid=0 2024-12-08T08:15:24,824 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:24,825 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/d01bc7a6746e4bc299ab6efb882e53a1 is 50, key is test_row_0/B:col10/1733645722986/Put/seqid=0 2024-12-08T08:15:24,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T08:15:24,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:24,833 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T08:15:24,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:24,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:24,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:24,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:24,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:24,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:24,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742142_1318 (size=13187) 2024-12-08T08:15:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/b57634bbe0574f04a31117d1412b37b1 is 50, key is test_row_0/A:col10/1733645723900/Put/seqid=0 2024-12-08T08:15:24,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742141_1317 (size=13187) 2024-12-08T08:15:24,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742143_1319 (size=12301) 2024-12-08T08:15:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T08:15:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:25,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645785086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645785086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:25,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645785151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:25,154 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in f652a2fb0fbb79b2d0f57e1ac321a224 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:25,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645785200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:25,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645785200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:25,248 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/d01bc7a6746e4bc299ab6efb882e53a1 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d01bc7a6746e4bc299ab6efb882e53a1 2024-12-08T08:15:25,253 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/B of f652a2fb0fbb79b2d0f57e1ac321a224 into d01bc7a6746e4bc299ab6efb882e53a1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:25,253 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:25,253 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/B, priority=13, startTime=1733645724811; duration=0sec 2024-12-08T08:15:25,253 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:25,253 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:B 2024-12-08T08:15:25,253 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:25,255 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:25,255 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/C is initiating minor compaction (all files) 2024-12-08T08:15:25,255 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/C in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:25,255 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/787e492105c642a7a6fc60ec0a60a9a3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/6a632a9fca244c2f8e47648a7b492f62, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/10cda65b971e42e9b51e38a0692a2ad0] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.8 K 2024-12-08T08:15:25,256 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 787e492105c642a7a6fc60ec0a60a9a3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733645722029 2024-12-08T08:15:25,256 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a632a9fca244c2f8e47648a7b492f62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733645722699 2024-12-08T08:15:25,256 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 10cda65b971e42e9b51e38a0692a2ad0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733645722976 2024-12-08T08:15:25,257 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/b57634bbe0574f04a31117d1412b37b1 2024-12-08T08:15:25,258 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/295c04bc0b834be9a127088cfae9b2d1 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/295c04bc0b834be9a127088cfae9b2d1 2024-12-08T08:15:25,265 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/A of f652a2fb0fbb79b2d0f57e1ac321a224 into 295c04bc0b834be9a127088cfae9b2d1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:25,265 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:25,265 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/A, priority=13, startTime=1733645724810; duration=0sec 2024-12-08T08:15:25,265 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:25,265 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:A 2024-12-08T08:15:25,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/aa5724f59cdb4105b5db0075e17169f4 is 50, key is test_row_0/B:col10/1733645723900/Put/seqid=0 2024-12-08T08:15:25,271 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#C#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:25,272 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/f326326cb2cb4abab3fdc08f285198ea is 50, key is test_row_0/C:col10/1733645722986/Put/seqid=0 2024-12-08T08:15:25,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742144_1320 (size=12301) 2024-12-08T08:15:25,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742145_1321 (size=13187) 2024-12-08T08:15:25,327 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/f326326cb2cb4abab3fdc08f285198ea as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/f326326cb2cb4abab3fdc08f285198ea 2024-12-08T08:15:25,332 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/C of f652a2fb0fbb79b2d0f57e1ac321a224 into f326326cb2cb4abab3fdc08f285198ea(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:25,332 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:25,332 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/C, priority=13, startTime=1733645724811; duration=0sec 2024-12-08T08:15:25,332 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:25,332 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:C 2024-12-08T08:15:25,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645785411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645785411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T08:15:25,699 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/aa5724f59cdb4105b5db0075e17169f4 2024-12-08T08:15:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/4f38661fb85a4104a3718935ecb1ed15 is 50, key is test_row_0/C:col10/1733645723900/Put/seqid=0 2024-12-08T08:15:25,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:25,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645785721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645785721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:25,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742146_1322 (size=12301) 2024-12-08T08:15:26,140 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/4f38661fb85a4104a3718935ecb1ed15 2024-12-08T08:15:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/b57634bbe0574f04a31117d1412b37b1 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b57634bbe0574f04a31117d1412b37b1 2024-12-08T08:15:26,168 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b57634bbe0574f04a31117d1412b37b1, entries=150, sequenceid=381, filesize=12.0 K 2024-12-08T08:15:26,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/aa5724f59cdb4105b5db0075e17169f4 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/aa5724f59cdb4105b5db0075e17169f4 2024-12-08T08:15:26,176 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/aa5724f59cdb4105b5db0075e17169f4, entries=150, sequenceid=381, filesize=12.0 K 2024-12-08T08:15:26,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/4f38661fb85a4104a3718935ecb1ed15 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/4f38661fb85a4104a3718935ecb1ed15 2024-12-08T08:15:26,186 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/4f38661fb85a4104a3718935ecb1ed15, entries=150, sequenceid=381, filesize=12.0 K 2024-12-08T08:15:26,189 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f652a2fb0fbb79b2d0f57e1ac321a224 in 1356ms, sequenceid=381, compaction requested=false 2024-12-08T08:15:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-08T08:15:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-08T08:15:26,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-08T08:15:26,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8160 sec 2024-12-08T08:15:26,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.8250 sec 2024-12-08T08:15:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:26,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T08:15:26,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:26,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:26,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:26,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:26,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:26,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:26,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/1d1e3c7aecd84ae2a9132c014421dba1 is 50, key is test_row_0/A:col10/1733645725055/Put/seqid=0 2024-12-08T08:15:26,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645786265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:26,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645786272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:26,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742147_1323 (size=12301) 2024-12-08T08:15:26,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645786381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645786381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T08:15:26,531 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:26,531 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-08T08:15:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T08:15:26,533 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:26,535 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:26,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:26,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645786590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:26,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645786591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T08:15:26,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/1d1e3c7aecd84ae2a9132c014421dba1 2024-12-08T08:15:26,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T08:15:26,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:26,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:26,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:26,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:26,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:26,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/992d5d7bd2b942c8b7f470772ef16fc0 is 50, key is test_row_0/B:col10/1733645725055/Put/seqid=0 2024-12-08T08:15:26,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742148_1324 (size=12301) 2024-12-08T08:15:26,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/992d5d7bd2b942c8b7f470772ef16fc0 2024-12-08T08:15:26,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/c5742568c2a64ec8af835b1b99966b1c is 50, key is test_row_0/C:col10/1733645725055/Put/seqid=0 2024-12-08T08:15:26,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742149_1325 (size=12301) 2024-12-08T08:15:26,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/c5742568c2a64ec8af835b1b99966b1c 2024-12-08T08:15:26,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/1d1e3c7aecd84ae2a9132c014421dba1 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1d1e3c7aecd84ae2a9132c014421dba1 2024-12-08T08:15:26,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1d1e3c7aecd84ae2a9132c014421dba1, entries=150, sequenceid=403, filesize=12.0 K 2024-12-08T08:15:26,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/992d5d7bd2b942c8b7f470772ef16fc0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/992d5d7bd2b942c8b7f470772ef16fc0 2024-12-08T08:15:26,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/992d5d7bd2b942c8b7f470772ef16fc0, entries=150, sequenceid=403, filesize=12.0 K 2024-12-08T08:15:26,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/c5742568c2a64ec8af835b1b99966b1c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/c5742568c2a64ec8af835b1b99966b1c 2024-12-08T08:15:26,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/c5742568c2a64ec8af835b1b99966b1c, entries=150, sequenceid=403, filesize=12.0 K 2024-12-08T08:15:26,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f652a2fb0fbb79b2d0f57e1ac321a224 in 594ms, sequenceid=403, compaction requested=true 2024-12-08T08:15:26,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:26,831 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:26,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:26,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:26,832 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:26,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:26,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:26,835 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:26,835 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:26,835 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/B is initiating minor compaction (all files) 2024-12-08T08:15:26,835 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/A is initiating minor compaction (all files) 2024-12-08T08:15:26,835 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/B in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:26,835 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/A in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:26,836 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d01bc7a6746e4bc299ab6efb882e53a1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/aa5724f59cdb4105b5db0075e17169f4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/992d5d7bd2b942c8b7f470772ef16fc0] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.9 K 2024-12-08T08:15:26,836 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/295c04bc0b834be9a127088cfae9b2d1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b57634bbe0574f04a31117d1412b37b1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1d1e3c7aecd84ae2a9132c014421dba1] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.9 K 2024-12-08T08:15:26,837 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting d01bc7a6746e4bc299ab6efb882e53a1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733645722976 2024-12-08T08:15:26,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:26,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:26,838 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting aa5724f59cdb4105b5db0075e17169f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733645723892 2024-12-08T08:15:26,838 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 295c04bc0b834be9a127088cfae9b2d1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733645722976 2024-12-08T08:15:26,838 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 992d5d7bd2b942c8b7f470772ef16fc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733645725055 2024-12-08T08:15:26,838 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting b57634bbe0574f04a31117d1412b37b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733645723892 2024-12-08T08:15:26,839 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d1e3c7aecd84ae2a9132c014421dba1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733645725055 2024-12-08T08:15:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T08:15:26,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T08:15:26,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:26,856 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T08:15:26,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:26,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:26,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:26,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:26,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:26,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:26,858 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#A#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:26,858 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/acba4c433b57424da1efbf79f470e175 is 50, key is test_row_0/A:col10/1733645725055/Put/seqid=0 2024-12-08T08:15:26,866 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#B#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:26,866 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/d937a22de3dd491588da46ebc35bc31a is 50, key is test_row_0/B:col10/1733645725055/Put/seqid=0 2024-12-08T08:15:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:26,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:26,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/934e0d21480640d5a04dbbee2f06e0b0 is 50, key is test_row_0/A:col10/1733645726246/Put/seqid=0 2024-12-08T08:15:26,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742150_1326 (size=13289) 2024-12-08T08:15:26,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742151_1327 (size=13289) 2024-12-08T08:15:26,922 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/d937a22de3dd491588da46ebc35bc31a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d937a22de3dd491588da46ebc35bc31a 2024-12-08T08:15:26,924 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/acba4c433b57424da1efbf79f470e175 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/acba4c433b57424da1efbf79f470e175 2024-12-08T08:15:26,932 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/B of f652a2fb0fbb79b2d0f57e1ac321a224 into d937a22de3dd491588da46ebc35bc31a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:26,933 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:26,933 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/B, priority=13, startTime=1733645726832; duration=0sec 2024-12-08T08:15:26,933 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:26,933 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:B 2024-12-08T08:15:26,933 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:26,935 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:26,935 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/C is initiating minor compaction (all files) 2024-12-08T08:15:26,935 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/A of f652a2fb0fbb79b2d0f57e1ac321a224 into acba4c433b57424da1efbf79f470e175(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:26,935 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:26,935 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/C in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:26,935 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/A, priority=13, startTime=1733645726831; duration=0sec 2024-12-08T08:15:26,935 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/f326326cb2cb4abab3fdc08f285198ea, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/4f38661fb85a4104a3718935ecb1ed15, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/c5742568c2a64ec8af835b1b99966b1c] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=36.9 K 2024-12-08T08:15:26,935 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:26,935 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:A 2024-12-08T08:15:26,936 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting f326326cb2cb4abab3fdc08f285198ea, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733645722976 2024-12-08T08:15:26,936 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f38661fb85a4104a3718935ecb1ed15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733645723892 2024-12-08T08:15:26,936 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting c5742568c2a64ec8af835b1b99966b1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733645725055 2024-12-08T08:15:26,951 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:26,952 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/de0ea27927c3437d9e38838f266c25ea is 50, key is test_row_0/C:col10/1733645725055/Put/seqid=0 2024-12-08T08:15:26,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742152_1328 (size=12301) 2024-12-08T08:15:26,958 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/934e0d21480640d5a04dbbee2f06e0b0 2024-12-08T08:15:26,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645786956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:26,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645786960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:26,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742153_1329 (size=13289) 2024-12-08T08:15:27,004 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/de0ea27927c3437d9e38838f266c25ea as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/de0ea27927c3437d9e38838f266c25ea 2024-12-08T08:15:27,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/34c4ae9c5ce844dfa5dcf2908f1527ca is 50, key is test_row_0/B:col10/1733645726246/Put/seqid=0 2024-12-08T08:15:27,014 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/C of f652a2fb0fbb79b2d0f57e1ac321a224 into de0ea27927c3437d9e38838f266c25ea(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:27,014 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:27,014 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/C, priority=13, startTime=1733645726837; duration=0sec 2024-12-08T08:15:27,015 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:27,015 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:C 2024-12-08T08:15:27,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742154_1330 (size=12301) 2024-12-08T08:15:27,028 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/34c4ae9c5ce844dfa5dcf2908f1527ca 2024-12-08T08:15:27,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/0c7a00fcddac42cf8d30bbf79e963965 is 50, key is test_row_0/C:col10/1733645726246/Put/seqid=0 2024-12-08T08:15:27,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:27,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645787071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645787072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:27,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742155_1331 (size=12301) 2024-12-08T08:15:27,082 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/0c7a00fcddac42cf8d30bbf79e963965 2024-12-08T08:15:27,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/934e0d21480640d5a04dbbee2f06e0b0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/934e0d21480640d5a04dbbee2f06e0b0 2024-12-08T08:15:27,096 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/934e0d21480640d5a04dbbee2f06e0b0, entries=150, sequenceid=416, filesize=12.0 K 2024-12-08T08:15:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/34c4ae9c5ce844dfa5dcf2908f1527ca as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/34c4ae9c5ce844dfa5dcf2908f1527ca 2024-12-08T08:15:27,105 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/34c4ae9c5ce844dfa5dcf2908f1527ca, entries=150, sequenceid=416, filesize=12.0 K 2024-12-08T08:15:27,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/0c7a00fcddac42cf8d30bbf79e963965 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/0c7a00fcddac42cf8d30bbf79e963965 2024-12-08T08:15:27,113 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/0c7a00fcddac42cf8d30bbf79e963965, entries=150, sequenceid=416, filesize=12.0 K 2024-12-08T08:15:27,114 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f652a2fb0fbb79b2d0f57e1ac321a224 in 258ms, sequenceid=416, compaction requested=false 2024-12-08T08:15:27,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:27,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:27,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-08T08:15:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-08T08:15:27,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-08T08:15:27,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 581 msec 2024-12-08T08:15:27,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 589 msec 2024-12-08T08:15:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T08:15:27,161 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:27,161 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-08T08:15:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T08:15:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:27,163 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:27,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T08:15:27,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:27,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:27,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:27,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:27,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:27,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:27,164 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:27,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:27,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/583e55dcaa7a481d8cd4cadf314de904 is 50, key is test_row_0/A:col10/1733645726948/Put/seqid=0 2024-12-08T08:15:27,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742156_1332 (size=14741) 2024-12-08T08:15:27,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645787200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T08:15:27,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:27,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645787281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645787281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:27,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:27,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645787310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:27,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-08T08:15:27,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:27,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:27,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:27,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:27,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:27,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-08T08:15:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:27,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T08:15:27,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645787520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:27,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/583e55dcaa7a481d8cd4cadf314de904 2024-12-08T08:15:27,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/b3cb80fef7bc4205bd116c9a765a31da is 50, key is test_row_0/B:col10/1733645726948/Put/seqid=0 2024-12-08T08:15:27,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742157_1333 (size=12301) 2024-12-08T08:15:27,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/b3cb80fef7bc4205bd116c9a765a31da 2024-12-08T08:15:27,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645787591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:27,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645787593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:27,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/acca19c717f14013b63aba88f8d65e3a is 50, key is test_row_0/C:col10/1733645726948/Put/seqid=0 2024-12-08T08:15:27,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-08T08:15:27,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:27,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:27,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:27,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:27,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:27,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742158_1334 (size=12301) 2024-12-08T08:15:27,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/acca19c717f14013b63aba88f8d65e3a 2024-12-08T08:15:27,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/583e55dcaa7a481d8cd4cadf314de904 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/583e55dcaa7a481d8cd4cadf314de904 2024-12-08T08:15:27,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/583e55dcaa7a481d8cd4cadf314de904, entries=200, sequenceid=439, filesize=14.4 K 2024-12-08T08:15:27,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/b3cb80fef7bc4205bd116c9a765a31da as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/b3cb80fef7bc4205bd116c9a765a31da 2024-12-08T08:15:27,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/b3cb80fef7bc4205bd116c9a765a31da, entries=150, sequenceid=439, filesize=12.0 K 2024-12-08T08:15:27,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/acca19c717f14013b63aba88f8d65e3a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/acca19c717f14013b63aba88f8d65e3a 2024-12-08T08:15:27,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/acca19c717f14013b63aba88f8d65e3a, entries=150, sequenceid=439, filesize=12.0 K 2024-12-08T08:15:27,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f652a2fb0fbb79b2d0f57e1ac321a224 in 500ms, sequenceid=439, compaction requested=true 2024-12-08T08:15:27,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:27,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:27,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:27,663 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:27,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:27,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:27,663 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:27,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:27,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:27,664 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:27,664 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:27,665 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/B is initiating minor compaction (all files) 2024-12-08T08:15:27,665 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/A is initiating minor compaction (all files) 2024-12-08T08:15:27,665 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/B in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:27,665 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/A in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:27,665 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d937a22de3dd491588da46ebc35bc31a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/34c4ae9c5ce844dfa5dcf2908f1527ca, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/b3cb80fef7bc4205bd116c9a765a31da] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=37.0 K 2024-12-08T08:15:27,665 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/acba4c433b57424da1efbf79f470e175, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/934e0d21480640d5a04dbbee2f06e0b0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/583e55dcaa7a481d8cd4cadf314de904] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=39.4 K 2024-12-08T08:15:27,665 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting d937a22de3dd491588da46ebc35bc31a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733645725055 2024-12-08T08:15:27,665 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting acba4c433b57424da1efbf79f470e175, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733645725055 2024-12-08T08:15:27,665 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 34c4ae9c5ce844dfa5dcf2908f1527ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733645726246 2024-12-08T08:15:27,671 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting b3cb80fef7bc4205bd116c9a765a31da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733645726947 2024-12-08T08:15:27,671 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 934e0d21480640d5a04dbbee2f06e0b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733645726246 2024-12-08T08:15:27,677 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 583e55dcaa7a481d8cd4cadf314de904, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733645726947 2024-12-08T08:15:27,690 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#B#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:27,691 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/d47c3021012f425b9591b4590b8132ba is 50, key is test_row_0/B:col10/1733645726948/Put/seqid=0 2024-12-08T08:15:27,694 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#A#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:27,695 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/677f4073eb8b45a1989397b97cea7ad7 is 50, key is test_row_0/A:col10/1733645726948/Put/seqid=0 2024-12-08T08:15:27,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742160_1336 (size=13391) 2024-12-08T08:15:27,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742159_1335 (size=13391) 2024-12-08T08:15:27,723 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/d47c3021012f425b9591b4590b8132ba as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d47c3021012f425b9591b4590b8132ba 2024-12-08T08:15:27,728 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/B of f652a2fb0fbb79b2d0f57e1ac321a224 into d47c3021012f425b9591b4590b8132ba(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:27,728 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:27,728 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/B, priority=13, startTime=1733645727663; duration=0sec 2024-12-08T08:15:27,728 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:27,728 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:B 2024-12-08T08:15:27,728 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:27,729 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:27,729 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/C is initiating minor compaction (all files) 2024-12-08T08:15:27,729 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/C in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:27,729 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/de0ea27927c3437d9e38838f266c25ea, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/0c7a00fcddac42cf8d30bbf79e963965, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/acca19c717f14013b63aba88f8d65e3a] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=37.0 K 2024-12-08T08:15:27,730 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting de0ea27927c3437d9e38838f266c25ea, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733645725055 2024-12-08T08:15:27,730 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c7a00fcddac42cf8d30bbf79e963965, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733645726246 2024-12-08T08:15:27,730 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting acca19c717f14013b63aba88f8d65e3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733645726947 2024-12-08T08:15:27,738 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#C#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:27,738 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/9cceca4805304874bd308fa2675a6346 is 50, key is test_row_0/C:col10/1733645726948/Put/seqid=0 2024-12-08T08:15:27,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742161_1337 (size=13391) 2024-12-08T08:15:27,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-08T08:15:27,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:27,777 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T08:15:27,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:27,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:27,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:27,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:27,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:27,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:27,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/3e3e43b30ee64bad8c07b879d9452fda is 50, key is test_row_0/A:col10/1733645727193/Put/seqid=0 2024-12-08T08:15:27,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742162_1338 (size=9857) 2024-12-08T08:15:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T08:15:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:27,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. as already flushing 2024-12-08T08:15:27,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645787878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:27,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645787991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:27,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:27,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:28,103 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/677f4073eb8b45a1989397b97cea7ad7 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/677f4073eb8b45a1989397b97cea7ad7 2024-12-08T08:15:28,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1733645788101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:28,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60676 deadline: 1733645788101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:28,107 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/A of f652a2fb0fbb79b2d0f57e1ac321a224 into 677f4073eb8b45a1989397b97cea7ad7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:28,107 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:28,107 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/A, priority=13, startTime=1733645727663; duration=0sec 2024-12-08T08:15:28,108 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:28,108 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:A 2024-12-08T08:15:28,151 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/9cceca4805304874bd308fa2675a6346 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/9cceca4805304874bd308fa2675a6346 2024-12-08T08:15:28,155 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/C of f652a2fb0fbb79b2d0f57e1ac321a224 into 9cceca4805304874bd308fa2675a6346(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:28,155 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:28,155 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/C, priority=13, startTime=1733645727663; duration=0sec 2024-12-08T08:15:28,155 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:28,155 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:C 2024-12-08T08:15:28,184 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/3e3e43b30ee64bad8c07b879d9452fda 2024-12-08T08:15:28,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/ddfb24fe35a74969b76f7c21b84ba952 is 50, key is test_row_0/B:col10/1733645727193/Put/seqid=0 2024-12-08T08:15:28,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742163_1339 (size=9857) 2024-12-08T08:15:28,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645788201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T08:15:28,338 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-12-08T08:15:28,338 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:28,338 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:28,338 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:28,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:28,338 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-12-08T08:15:28,338 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:28,338 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:28,338 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:28,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:28,339 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-12-08T08:15:28,339 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:28,339 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:28,339 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:28,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:28,340 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-12-08T08:15:28,340 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:28,340 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:28,340 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-12-08T08:15:28,340 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:28,340 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:28,340 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:28,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:28,341 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:28,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:28,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645788511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:28,602 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/ddfb24fe35a74969b76f7c21b84ba952 2024-12-08T08:15:28,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/331356270215435c939431aaac490e55 is 50, key is test_row_0/C:col10/1733645727193/Put/seqid=0 2024-12-08T08:15:28,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742164_1340 (size=9857) 2024-12-08T08:15:29,019 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/331356270215435c939431aaac490e55 2024-12-08T08:15:29,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60734 deadline: 1733645789020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:29,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:15:29,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/3e3e43b30ee64bad8c07b879d9452fda as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/3e3e43b30ee64bad8c07b879d9452fda 2024-12-08T08:15:29,029 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/3e3e43b30ee64bad8c07b879d9452fda, entries=100, sequenceid=452, filesize=9.6 K 2024-12-08T08:15:29,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/ddfb24fe35a74969b76f7c21b84ba952 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/ddfb24fe35a74969b76f7c21b84ba952 2024-12-08T08:15:29,034 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/ddfb24fe35a74969b76f7c21b84ba952, entries=100, sequenceid=452, filesize=9.6 K 2024-12-08T08:15:29,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/331356270215435c939431aaac490e55 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/331356270215435c939431aaac490e55 2024-12-08T08:15:29,038 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/331356270215435c939431aaac490e55, entries=100, sequenceid=452, filesize=9.6 K 2024-12-08T08:15:29,039 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f652a2fb0fbb79b2d0f57e1ac321a224 in 1263ms, sequenceid=452, compaction requested=false 2024-12-08T08:15:29,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:29,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:29,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-08T08:15:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-08T08:15:29,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-08T08:15:29,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8770 sec 2024-12-08T08:15:29,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.8820 sec 2024-12-08T08:15:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:29,112 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-12-08T08:15:29,112 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:29,112 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:29,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:29,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-08T08:15:29,112 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-12-08T08:15:29,112 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:29,112 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:29,113 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:29,113 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:29,113 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:29,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:29,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:29,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:29,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:29,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:29,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:29,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/c4a6b38f1d2845baab29463bdde0960f is 50, key is test_row_0/A:col10/1733645729110/Put/seqid=0 2024-12-08T08:15:29,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742165_1341 (size=12301) 2024-12-08T08:15:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T08:15:29,320 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:29,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/c4a6b38f1d2845baab29463bdde0960f 2024-12-08T08:15:29,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/fd688bba2c594a65a4baa65fcb6ac9f9 is 50, key is test_row_0/B:col10/1733645729110/Put/seqid=0 2024-12-08T08:15:29,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742166_1342 (size=12301) 2024-12-08T08:15:29,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/fd688bba2c594a65a4baa65fcb6ac9f9 2024-12-08T08:15:29,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/8a548f1fca2245fc862418223821b763 is 50, key is test_row_0/C:col10/1733645729110/Put/seqid=0 2024-12-08T08:15:29,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742167_1343 (size=12301) 2024-12-08T08:15:30,032 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-12-08T08:15:30,032 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:30,032 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:30,032 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:30,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:30,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/8a548f1fca2245fc862418223821b763 2024-12-08T08:15:30,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/c4a6b38f1d2845baab29463bdde0960f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/c4a6b38f1d2845baab29463bdde0960f 2024-12-08T08:15:30,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/c4a6b38f1d2845baab29463bdde0960f, entries=150, sequenceid=476, filesize=12.0 K 2024-12-08T08:15:30,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/fd688bba2c594a65a4baa65fcb6ac9f9 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/fd688bba2c594a65a4baa65fcb6ac9f9 2024-12-08T08:15:30,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/fd688bba2c594a65a4baa65fcb6ac9f9, entries=150, sequenceid=476, filesize=12.0 K 2024-12-08T08:15:30,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/8a548f1fca2245fc862418223821b763 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8a548f1fca2245fc862418223821b763 2024-12-08T08:15:30,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8a548f1fca2245fc862418223821b763, entries=150, sequenceid=476, filesize=12.0 K 2024-12-08T08:15:30,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=6.71 KB/6870 for f652a2fb0fbb79b2d0f57e1ac321a224 in 1253ms, sequenceid=476, compaction requested=true 2024-12-08T08:15:30,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:30,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:30,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:30,365 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:30,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:30,366 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f652a2fb0fbb79b2d0f57e1ac321a224:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:30,367 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:30,367 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:30,367 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/A is initiating minor compaction (all files) 2024-12-08T08:15:30,367 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/B is initiating minor compaction (all files) 2024-12-08T08:15:30,367 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/B in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:30,367 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/A in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:30,367 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/677f4073eb8b45a1989397b97cea7ad7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/3e3e43b30ee64bad8c07b879d9452fda, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/c4a6b38f1d2845baab29463bdde0960f] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=34.7 K 2024-12-08T08:15:30,367 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d47c3021012f425b9591b4590b8132ba, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/ddfb24fe35a74969b76f7c21b84ba952, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/fd688bba2c594a65a4baa65fcb6ac9f9] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=34.7 K 2024-12-08T08:15:30,367 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 677f4073eb8b45a1989397b97cea7ad7, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733645726947 2024-12-08T08:15:30,367 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting d47c3021012f425b9591b4590b8132ba, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733645726947 2024-12-08T08:15:30,367 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting ddfb24fe35a74969b76f7c21b84ba952, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733645727193 2024-12-08T08:15:30,367 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e3e43b30ee64bad8c07b879d9452fda, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733645727193 2024-12-08T08:15:30,368 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting fd688bba2c594a65a4baa65fcb6ac9f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733645727871 2024-12-08T08:15:30,369 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4a6b38f1d2845baab29463bdde0960f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733645727871 2024-12-08T08:15:30,375 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#B#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:30,375 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#A#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:30,376 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/b9b23f7d34b84eb5807fc9d1a54c5be5 is 50, key is test_row_0/A:col10/1733645729110/Put/seqid=0 2024-12-08T08:15:30,376 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/de369d703f7f44f28660acffa9561238 is 50, key is test_row_0/B:col10/1733645729110/Put/seqid=0 2024-12-08T08:15:30,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742168_1344 (size=13493) 2024-12-08T08:15:30,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742169_1345 (size=13493) 2024-12-08T08:15:30,385 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/de369d703f7f44f28660acffa9561238 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/de369d703f7f44f28660acffa9561238 2024-12-08T08:15:30,388 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/B of f652a2fb0fbb79b2d0f57e1ac321a224 into de369d703f7f44f28660acffa9561238(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:30,388 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:30,388 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/B, priority=13, startTime=1733645730365; duration=0sec 2024-12-08T08:15:30,388 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:30,388 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:B 2024-12-08T08:15:30,388 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:30,389 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:30,389 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): f652a2fb0fbb79b2d0f57e1ac321a224/C is initiating minor compaction (all files) 2024-12-08T08:15:30,389 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f652a2fb0fbb79b2d0f57e1ac321a224/C in TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:30,389 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/9cceca4805304874bd308fa2675a6346, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/331356270215435c939431aaac490e55, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8a548f1fca2245fc862418223821b763] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp, totalSize=34.7 K 2024-12-08T08:15:30,390 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cceca4805304874bd308fa2675a6346, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733645726947 2024-12-08T08:15:30,390 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 331356270215435c939431aaac490e55, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733645727193 2024-12-08T08:15:30,390 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a548f1fca2245fc862418223821b763, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733645727871 2024-12-08T08:15:30,395 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f652a2fb0fbb79b2d0f57e1ac321a224#C#compaction#299 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:30,396 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/4c82c6147df54cc4a4529d2cc982c59a is 50, key is test_row_0/C:col10/1733645729110/Put/seqid=0 2024-12-08T08:15:30,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742170_1346 (size=13493) 2024-12-08T08:15:30,407 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T08:15:30,790 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/b9b23f7d34b84eb5807fc9d1a54c5be5 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b9b23f7d34b84eb5807fc9d1a54c5be5 2024-12-08T08:15:30,794 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/A of f652a2fb0fbb79b2d0f57e1ac321a224 into b9b23f7d34b84eb5807fc9d1a54c5be5(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:30,794 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:30,794 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/A, priority=13, startTime=1733645730365; duration=0sec 2024-12-08T08:15:30,794 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:30,794 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:A 2024-12-08T08:15:30,803 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/4c82c6147df54cc4a4529d2cc982c59a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/4c82c6147df54cc4a4529d2cc982c59a 2024-12-08T08:15:30,808 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f652a2fb0fbb79b2d0f57e1ac321a224/C of f652a2fb0fbb79b2d0f57e1ac321a224 into 4c82c6147df54cc4a4529d2cc982c59a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:30,808 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f652a2fb0fbb79b2d0f57e1ac321a224: 2024-12-08T08:15:30,808 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., storeName=f652a2fb0fbb79b2d0f57e1ac321a224/C, priority=13, startTime=1733645730366; duration=0sec 2024-12-08T08:15:30,808 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:30,808 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f652a2fb0fbb79b2d0f57e1ac321a224:C 2024-12-08T08:15:32,255 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-12-08T08:15:32,255 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:32,255 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:32,255 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:32,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:32,364 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-12-08T08:15:32,364 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:32,364 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:32,365 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:32,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:32,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T08:15:32,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-12-08T08:15:32,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-12-08T08:15:32,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-12-08T08:15:32,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-12-08T08:15:32,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-12-08T08:15:32,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T08:15:32,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3919 2024-12-08T08:15:32,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3849 2024-12-08T08:15:32,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3943 2024-12-08T08:15:32,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3889 2024-12-08T08:15:32,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3848 2024-12-08T08:15:32,366 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T08:15:32,366 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T08:15:32,366 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T08:15:32,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:32,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:32,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:32,368 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-08T08:15:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T08:15:32,373 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645732373"}]},"ts":"1733645732373"} 2024-12-08T08:15:32,375 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T08:15:32,375 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T08:15:32,376 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T08:15:32,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f652a2fb0fbb79b2d0f57e1ac321a224, UNASSIGN}] 2024-12-08T08:15:32,378 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f652a2fb0fbb79b2d0f57e1ac321a224, UNASSIGN 2024-12-08T08:15:32,379 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=f652a2fb0fbb79b2d0f57e1ac321a224, regionState=CLOSING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:15:32,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f652a2fb0fbb79b2d0f57e1ac321a224, UNASSIGN because future has completed 2024-12-08T08:15:32,380 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T08:15:32,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:15:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T08:15:32,532 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:32,532 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T08:15:32,533 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing f652a2fb0fbb79b2d0f57e1ac321a224, disabling compactions & flushes 2024-12-08T08:15:32,533 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:32,533 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:32,533 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. after waiting 0 ms 2024-12-08T08:15:32,533 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:32,533 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing f652a2fb0fbb79b2d0f57e1ac321a224 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-08T08:15:32,533 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=A 2024-12-08T08:15:32,533 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:32,533 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=B 2024-12-08T08:15:32,534 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:32,534 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f652a2fb0fbb79b2d0f57e1ac321a224, store=C 2024-12-08T08:15:32,534 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:32,540 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/5568e9edfb414229801152e9b0509333 is 50, key is test_row_1/A:col10/1733645732251/Put/seqid=0 2024-12-08T08:15:32,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742171_1347 (size=9857) 2024-12-08T08:15:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T08:15:32,947 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/5568e9edfb414229801152e9b0509333 2024-12-08T08:15:32,954 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/2a9c939cdd914be589e774137df64276 is 50, key is test_row_1/B:col10/1733645732251/Put/seqid=0 2024-12-08T08:15:32,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742172_1348 (size=9857) 2024-12-08T08:15:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T08:15:33,362 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/2a9c939cdd914be589e774137df64276 2024-12-08T08:15:33,369 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/a09a5ca46abb412eb36f695257f6af6e is 50, key is test_row_1/C:col10/1733645732251/Put/seqid=0 2024-12-08T08:15:33,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742173_1349 (size=9857) 2024-12-08T08:15:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T08:15:33,773 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/a09a5ca46abb412eb36f695257f6af6e 2024-12-08T08:15:33,779 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/A/5568e9edfb414229801152e9b0509333 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/5568e9edfb414229801152e9b0509333 2024-12-08T08:15:33,785 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/5568e9edfb414229801152e9b0509333, entries=100, sequenceid=485, filesize=9.6 K 2024-12-08T08:15:33,786 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/B/2a9c939cdd914be589e774137df64276 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/2a9c939cdd914be589e774137df64276 2024-12-08T08:15:33,789 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/2a9c939cdd914be589e774137df64276, entries=100, sequenceid=485, filesize=9.6 K 2024-12-08T08:15:33,790 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/.tmp/C/a09a5ca46abb412eb36f695257f6af6e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/a09a5ca46abb412eb36f695257f6af6e 2024-12-08T08:15:33,793 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/a09a5ca46abb412eb36f695257f6af6e, entries=100, sequenceid=485, filesize=9.6 K 2024-12-08T08:15:33,794 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for f652a2fb0fbb79b2d0f57e1ac321a224 in 1261ms, sequenceid=485, compaction requested=false 2024-12-08T08:15:33,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/3cae5893469d494db7999a77e0ae8084, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/382e45f6c9d34a66acf0a374e99cb770, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/f0eb9bde7c50423aacfd1d90c7dd9202, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4b2e82a70900455dad65caef4d089b3a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/084df6063e924bb19d9b227c3865d216, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/76064382fb45402ba5755651c9418ae4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1d4e7d950f21442696102474447e359c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4561df2f2c4946568952b7690e9bb069, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/08b779602dcc4b5593bfdff8b81c1243, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1502157682034aaf949e250316bbe8c8, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/07974b0d77754db68900b908fc999133, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/a637347519054c968e6406c5bd2523c6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/c4dd69176de74a15b6b2de27c8f5e4dc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/6fa915853f3e4703a354fb15c82135b3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/0e0ca28981754eedb28155749ac7b6e6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/14f840ba5fcc41c391c0204789090430, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/cbc77e2195dc443fb560f3fbc780fd87, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/05cb7122e91b4f5b9a5ffb6c84af4a2b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4d38f3d5ac2848b98c59130f95c26060, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/765229b428934e2495def6bd7ae86ea6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/e1789b4833c44019a7eef5aa80293ca2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/185e7a03b6d54767acd6d585d72047cf, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/fdc5ed286c7e40d38efd5de8983d5b0e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b7069d83703648ce814e5933f2e14353, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/9f25274e04404722b0d4ade9ba24d256, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/295c04bc0b834be9a127088cfae9b2d1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/8032295376b146bd99c3cab5c104a4ca, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b57634bbe0574f04a31117d1412b37b1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/acba4c433b57424da1efbf79f470e175, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1d1e3c7aecd84ae2a9132c014421dba1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/934e0d21480640d5a04dbbee2f06e0b0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/583e55dcaa7a481d8cd4cadf314de904, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/677f4073eb8b45a1989397b97cea7ad7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/3e3e43b30ee64bad8c07b879d9452fda, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/c4a6b38f1d2845baab29463bdde0960f] to archive 2024-12-08T08:15:33,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:15:33,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/3cae5893469d494db7999a77e0ae8084 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/3cae5893469d494db7999a77e0ae8084 2024-12-08T08:15:33,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/382e45f6c9d34a66acf0a374e99cb770 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/382e45f6c9d34a66acf0a374e99cb770 2024-12-08T08:15:33,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/f0eb9bde7c50423aacfd1d90c7dd9202 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/f0eb9bde7c50423aacfd1d90c7dd9202 2024-12-08T08:15:33,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4b2e82a70900455dad65caef4d089b3a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4b2e82a70900455dad65caef4d089b3a 2024-12-08T08:15:33,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/084df6063e924bb19d9b227c3865d216 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/084df6063e924bb19d9b227c3865d216 2024-12-08T08:15:33,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/76064382fb45402ba5755651c9418ae4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/76064382fb45402ba5755651c9418ae4 2024-12-08T08:15:33,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1d4e7d950f21442696102474447e359c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1d4e7d950f21442696102474447e359c 2024-12-08T08:15:33,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4561df2f2c4946568952b7690e9bb069 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4561df2f2c4946568952b7690e9bb069 2024-12-08T08:15:33,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/08b779602dcc4b5593bfdff8b81c1243 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/08b779602dcc4b5593bfdff8b81c1243 2024-12-08T08:15:33,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1502157682034aaf949e250316bbe8c8 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1502157682034aaf949e250316bbe8c8 2024-12-08T08:15:33,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/07974b0d77754db68900b908fc999133 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/07974b0d77754db68900b908fc999133 2024-12-08T08:15:33,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/a637347519054c968e6406c5bd2523c6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/a637347519054c968e6406c5bd2523c6 2024-12-08T08:15:33,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/c4dd69176de74a15b6b2de27c8f5e4dc to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/c4dd69176de74a15b6b2de27c8f5e4dc 2024-12-08T08:15:33,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/6fa915853f3e4703a354fb15c82135b3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/6fa915853f3e4703a354fb15c82135b3 2024-12-08T08:15:33,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/0e0ca28981754eedb28155749ac7b6e6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/0e0ca28981754eedb28155749ac7b6e6 2024-12-08T08:15:33,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/14f840ba5fcc41c391c0204789090430 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/14f840ba5fcc41c391c0204789090430 2024-12-08T08:15:33,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/cbc77e2195dc443fb560f3fbc780fd87 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/cbc77e2195dc443fb560f3fbc780fd87 2024-12-08T08:15:33,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/05cb7122e91b4f5b9a5ffb6c84af4a2b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/05cb7122e91b4f5b9a5ffb6c84af4a2b 2024-12-08T08:15:33,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4d38f3d5ac2848b98c59130f95c26060 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/4d38f3d5ac2848b98c59130f95c26060 2024-12-08T08:15:33,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/765229b428934e2495def6bd7ae86ea6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/765229b428934e2495def6bd7ae86ea6 2024-12-08T08:15:33,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/e1789b4833c44019a7eef5aa80293ca2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/e1789b4833c44019a7eef5aa80293ca2 2024-12-08T08:15:33,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/185e7a03b6d54767acd6d585d72047cf to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/185e7a03b6d54767acd6d585d72047cf 2024-12-08T08:15:33,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/fdc5ed286c7e40d38efd5de8983d5b0e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/fdc5ed286c7e40d38efd5de8983d5b0e 2024-12-08T08:15:33,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b7069d83703648ce814e5933f2e14353 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b7069d83703648ce814e5933f2e14353 2024-12-08T08:15:33,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/9f25274e04404722b0d4ade9ba24d256 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/9f25274e04404722b0d4ade9ba24d256 2024-12-08T08:15:33,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/295c04bc0b834be9a127088cfae9b2d1 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/295c04bc0b834be9a127088cfae9b2d1 2024-12-08T08:15:33,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/8032295376b146bd99c3cab5c104a4ca to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/8032295376b146bd99c3cab5c104a4ca 2024-12-08T08:15:33,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b57634bbe0574f04a31117d1412b37b1 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b57634bbe0574f04a31117d1412b37b1 2024-12-08T08:15:33,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/acba4c433b57424da1efbf79f470e175 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/acba4c433b57424da1efbf79f470e175 2024-12-08T08:15:33,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1d1e3c7aecd84ae2a9132c014421dba1 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/1d1e3c7aecd84ae2a9132c014421dba1 2024-12-08T08:15:33,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/934e0d21480640d5a04dbbee2f06e0b0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/934e0d21480640d5a04dbbee2f06e0b0 2024-12-08T08:15:33,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/583e55dcaa7a481d8cd4cadf314de904 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/583e55dcaa7a481d8cd4cadf314de904 2024-12-08T08:15:33,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/677f4073eb8b45a1989397b97cea7ad7 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/677f4073eb8b45a1989397b97cea7ad7 2024-12-08T08:15:33,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/3e3e43b30ee64bad8c07b879d9452fda to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/3e3e43b30ee64bad8c07b879d9452fda 2024-12-08T08:15:33,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/c4a6b38f1d2845baab29463bdde0960f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/c4a6b38f1d2845baab29463bdde0960f 2024-12-08T08:15:33,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/dec9feb069114e68a34847abb46093c3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/8617170716d44237b88d4c66e79e3244, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/75db671f249d4f24876100253a15f4e5, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/e25d4481317a4f178530187e47731c38, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/eebbc795ecb24ff1874786cfe794e3c6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/549d47fa81b34542b86847326b50f454, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/80e6e723c5dd4ddb83a2880806e4d1b0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/93d9491fd6a44648870baad7c2df7414, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/9c71fe68c53a429086c75aa9768f023c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/b768595a582b4b13b0d487169fc80738, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/8db62a11f5e24eb5a15e2f9d9c48c816, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a665a17c6aa44e128035de77476f3eff, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/4fe104b7baea42d9b9950ffe95170988, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a0354cad0a964e79aa2274e022c444e2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a48b2a9c27ac444583f4fcfe3312e8fb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/6a226dedd16d4a35adc47cbe76702e44, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/f3d84f00738d47549dfe6e5c3d6dd879, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c21e120fa7f349caab5a05e6800ae104, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/250ee7918f5c4676ae1754e2260c4e9d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/422dc58d870247b598dc257c65771403, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/702743e73da0440387af700f6b926554, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c99f1023569b4ba18125ba8600aeb654, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c77b2225ee81499aaa3f24af4df7deda, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/20ea1320dedd4be6b4f8eda2cf01075e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/90c174741e514a40a3a8241626de6197, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d01bc7a6746e4bc299ab6efb882e53a1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/829bbaa975d34101bdf9c61c509aaa0b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/aa5724f59cdb4105b5db0075e17169f4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d937a22de3dd491588da46ebc35bc31a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/992d5d7bd2b942c8b7f470772ef16fc0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/34c4ae9c5ce844dfa5dcf2908f1527ca, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d47c3021012f425b9591b4590b8132ba, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/b3cb80fef7bc4205bd116c9a765a31da, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/ddfb24fe35a74969b76f7c21b84ba952, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/fd688bba2c594a65a4baa65fcb6ac9f9] to archive 2024-12-08T08:15:33,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:15:33,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/dec9feb069114e68a34847abb46093c3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/dec9feb069114e68a34847abb46093c3 2024-12-08T08:15:33,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/8617170716d44237b88d4c66e79e3244 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/8617170716d44237b88d4c66e79e3244 2024-12-08T08:15:33,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/75db671f249d4f24876100253a15f4e5 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/75db671f249d4f24876100253a15f4e5 2024-12-08T08:15:33,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/e25d4481317a4f178530187e47731c38 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/e25d4481317a4f178530187e47731c38 2024-12-08T08:15:33,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/eebbc795ecb24ff1874786cfe794e3c6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/eebbc795ecb24ff1874786cfe794e3c6 2024-12-08T08:15:33,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/549d47fa81b34542b86847326b50f454 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/549d47fa81b34542b86847326b50f454 2024-12-08T08:15:33,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/80e6e723c5dd4ddb83a2880806e4d1b0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/80e6e723c5dd4ddb83a2880806e4d1b0 2024-12-08T08:15:33,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/93d9491fd6a44648870baad7c2df7414 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/93d9491fd6a44648870baad7c2df7414 2024-12-08T08:15:33,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/9c71fe68c53a429086c75aa9768f023c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/9c71fe68c53a429086c75aa9768f023c 2024-12-08T08:15:33,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/b768595a582b4b13b0d487169fc80738 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/b768595a582b4b13b0d487169fc80738 2024-12-08T08:15:33,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/8db62a11f5e24eb5a15e2f9d9c48c816 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/8db62a11f5e24eb5a15e2f9d9c48c816 2024-12-08T08:15:33,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a665a17c6aa44e128035de77476f3eff to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a665a17c6aa44e128035de77476f3eff 2024-12-08T08:15:33,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/4fe104b7baea42d9b9950ffe95170988 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/4fe104b7baea42d9b9950ffe95170988 2024-12-08T08:15:33,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a0354cad0a964e79aa2274e022c444e2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a0354cad0a964e79aa2274e022c444e2 2024-12-08T08:15:33,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a48b2a9c27ac444583f4fcfe3312e8fb to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/a48b2a9c27ac444583f4fcfe3312e8fb 2024-12-08T08:15:33,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/6a226dedd16d4a35adc47cbe76702e44 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/6a226dedd16d4a35adc47cbe76702e44 2024-12-08T08:15:33,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/f3d84f00738d47549dfe6e5c3d6dd879 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/f3d84f00738d47549dfe6e5c3d6dd879 2024-12-08T08:15:33,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c21e120fa7f349caab5a05e6800ae104 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c21e120fa7f349caab5a05e6800ae104 2024-12-08T08:15:33,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/250ee7918f5c4676ae1754e2260c4e9d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/250ee7918f5c4676ae1754e2260c4e9d 2024-12-08T08:15:33,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/422dc58d870247b598dc257c65771403 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/422dc58d870247b598dc257c65771403 2024-12-08T08:15:33,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/702743e73da0440387af700f6b926554 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/702743e73da0440387af700f6b926554 2024-12-08T08:15:33,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c99f1023569b4ba18125ba8600aeb654 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c99f1023569b4ba18125ba8600aeb654 2024-12-08T08:15:33,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c77b2225ee81499aaa3f24af4df7deda to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/c77b2225ee81499aaa3f24af4df7deda 2024-12-08T08:15:33,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/20ea1320dedd4be6b4f8eda2cf01075e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/20ea1320dedd4be6b4f8eda2cf01075e 2024-12-08T08:15:33,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/90c174741e514a40a3a8241626de6197 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/90c174741e514a40a3a8241626de6197 2024-12-08T08:15:33,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d01bc7a6746e4bc299ab6efb882e53a1 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d01bc7a6746e4bc299ab6efb882e53a1 2024-12-08T08:15:33,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/829bbaa975d34101bdf9c61c509aaa0b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/829bbaa975d34101bdf9c61c509aaa0b 2024-12-08T08:15:33,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/aa5724f59cdb4105b5db0075e17169f4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/aa5724f59cdb4105b5db0075e17169f4 2024-12-08T08:15:33,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d937a22de3dd491588da46ebc35bc31a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d937a22de3dd491588da46ebc35bc31a 2024-12-08T08:15:33,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/992d5d7bd2b942c8b7f470772ef16fc0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/992d5d7bd2b942c8b7f470772ef16fc0 2024-12-08T08:15:33,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/34c4ae9c5ce844dfa5dcf2908f1527ca to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/34c4ae9c5ce844dfa5dcf2908f1527ca 2024-12-08T08:15:33,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d47c3021012f425b9591b4590b8132ba to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/d47c3021012f425b9591b4590b8132ba 2024-12-08T08:15:33,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/b3cb80fef7bc4205bd116c9a765a31da to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/b3cb80fef7bc4205bd116c9a765a31da 2024-12-08T08:15:33,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/ddfb24fe35a74969b76f7c21b84ba952 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/ddfb24fe35a74969b76f7c21b84ba952 2024-12-08T08:15:33,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/fd688bba2c594a65a4baa65fcb6ac9f9 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/fd688bba2c594a65a4baa65fcb6ac9f9 2024-12-08T08:15:33,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/919f3e602c904932b6aa26df5d79ca4c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/5072fb5b8e7b4bb5b305f2584c6cf6cb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/3b681692de5f4f9d930152b734d3a93e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/e8e423bca62f447ca872b4bb11e90aca, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2dfe3f8fc60c49ecbf64a1305e3933bc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8cadfc6d601d40c4a88b864eb7fedbcd, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/ff98330c2c0a417990d0b02935c941c6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/6efaaf08210d483ebc63875fceed7fbf, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2a6ca539455344bf8836fcfd79f83f0c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/815be12c3cbf4b53a75d042482feb328, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2af4cd6e9d66408f89ad76e5d68e7d7c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/e637d38476594cae81e06d33d1aa1cf3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/0f24210e2c0a48ecb8c0998192894aaf, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/a18434af0ef64d3e90e78738b87cd723, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/c351251b253b48cb8178b566672222e4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8af2687d91434725a4a07788d9fa1428, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/aa24fea34bbd42e8b5ce861184a5572e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/05c379a3abd247bea4bb497b20e6f580, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/48d267e0742542eebe5a1a33c2b01c41, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/ea8535702ae14594a6d1e2c9a461b642, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/48a458d49e834072addf30832961f457, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/063730bde9e64fbbb31b10bc3d6a8200, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/787e492105c642a7a6fc60ec0a60a9a3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/d530ba7569df40ddbaacd77bfbb61b6d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/6a632a9fca244c2f8e47648a7b492f62, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/f326326cb2cb4abab3fdc08f285198ea, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/10cda65b971e42e9b51e38a0692a2ad0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/4f38661fb85a4104a3718935ecb1ed15, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/de0ea27927c3437d9e38838f266c25ea, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/c5742568c2a64ec8af835b1b99966b1c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/0c7a00fcddac42cf8d30bbf79e963965, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/9cceca4805304874bd308fa2675a6346, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/acca19c717f14013b63aba88f8d65e3a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/331356270215435c939431aaac490e55, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8a548f1fca2245fc862418223821b763] to archive 2024-12-08T08:15:33,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:15:33,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/919f3e602c904932b6aa26df5d79ca4c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/919f3e602c904932b6aa26df5d79ca4c 2024-12-08T08:15:33,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/5072fb5b8e7b4bb5b305f2584c6cf6cb to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/5072fb5b8e7b4bb5b305f2584c6cf6cb 2024-12-08T08:15:33,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/3b681692de5f4f9d930152b734d3a93e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/3b681692de5f4f9d930152b734d3a93e 2024-12-08T08:15:33,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/e8e423bca62f447ca872b4bb11e90aca to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/e8e423bca62f447ca872b4bb11e90aca 2024-12-08T08:15:33,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2dfe3f8fc60c49ecbf64a1305e3933bc to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2dfe3f8fc60c49ecbf64a1305e3933bc 2024-12-08T08:15:33,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8cadfc6d601d40c4a88b864eb7fedbcd to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8cadfc6d601d40c4a88b864eb7fedbcd 2024-12-08T08:15:33,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/ff98330c2c0a417990d0b02935c941c6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/ff98330c2c0a417990d0b02935c941c6 2024-12-08T08:15:33,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/6efaaf08210d483ebc63875fceed7fbf to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/6efaaf08210d483ebc63875fceed7fbf 2024-12-08T08:15:33,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2a6ca539455344bf8836fcfd79f83f0c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2a6ca539455344bf8836fcfd79f83f0c 2024-12-08T08:15:33,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/815be12c3cbf4b53a75d042482feb328 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/815be12c3cbf4b53a75d042482feb328 2024-12-08T08:15:33,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2af4cd6e9d66408f89ad76e5d68e7d7c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/2af4cd6e9d66408f89ad76e5d68e7d7c 2024-12-08T08:15:33,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/e637d38476594cae81e06d33d1aa1cf3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/e637d38476594cae81e06d33d1aa1cf3 2024-12-08T08:15:33,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/0f24210e2c0a48ecb8c0998192894aaf to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/0f24210e2c0a48ecb8c0998192894aaf 2024-12-08T08:15:33,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/a18434af0ef64d3e90e78738b87cd723 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/a18434af0ef64d3e90e78738b87cd723 2024-12-08T08:15:33,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/c351251b253b48cb8178b566672222e4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/c351251b253b48cb8178b566672222e4 2024-12-08T08:15:33,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8af2687d91434725a4a07788d9fa1428 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8af2687d91434725a4a07788d9fa1428 2024-12-08T08:15:33,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/aa24fea34bbd42e8b5ce861184a5572e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/aa24fea34bbd42e8b5ce861184a5572e 2024-12-08T08:15:33,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/05c379a3abd247bea4bb497b20e6f580 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/05c379a3abd247bea4bb497b20e6f580 2024-12-08T08:15:33,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/48d267e0742542eebe5a1a33c2b01c41 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/48d267e0742542eebe5a1a33c2b01c41 2024-12-08T08:15:33,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/ea8535702ae14594a6d1e2c9a461b642 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/ea8535702ae14594a6d1e2c9a461b642 2024-12-08T08:15:33,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/48a458d49e834072addf30832961f457 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/48a458d49e834072addf30832961f457 2024-12-08T08:15:33,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/063730bde9e64fbbb31b10bc3d6a8200 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/063730bde9e64fbbb31b10bc3d6a8200 2024-12-08T08:15:33,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/787e492105c642a7a6fc60ec0a60a9a3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/787e492105c642a7a6fc60ec0a60a9a3 2024-12-08T08:15:33,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/d530ba7569df40ddbaacd77bfbb61b6d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/d530ba7569df40ddbaacd77bfbb61b6d 2024-12-08T08:15:33,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/6a632a9fca244c2f8e47648a7b492f62 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/6a632a9fca244c2f8e47648a7b492f62 2024-12-08T08:15:33,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/f326326cb2cb4abab3fdc08f285198ea to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/f326326cb2cb4abab3fdc08f285198ea 2024-12-08T08:15:33,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/10cda65b971e42e9b51e38a0692a2ad0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/10cda65b971e42e9b51e38a0692a2ad0 2024-12-08T08:15:33,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/4f38661fb85a4104a3718935ecb1ed15 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/4f38661fb85a4104a3718935ecb1ed15 2024-12-08T08:15:33,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/de0ea27927c3437d9e38838f266c25ea to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/de0ea27927c3437d9e38838f266c25ea 2024-12-08T08:15:33,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/c5742568c2a64ec8af835b1b99966b1c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/c5742568c2a64ec8af835b1b99966b1c 2024-12-08T08:15:33,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/0c7a00fcddac42cf8d30bbf79e963965 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/0c7a00fcddac42cf8d30bbf79e963965 2024-12-08T08:15:33,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/9cceca4805304874bd308fa2675a6346 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/9cceca4805304874bd308fa2675a6346 2024-12-08T08:15:33,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/acca19c717f14013b63aba88f8d65e3a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/acca19c717f14013b63aba88f8d65e3a 2024-12-08T08:15:33,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/331356270215435c939431aaac490e55 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/331356270215435c939431aaac490e55 2024-12-08T08:15:33,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8a548f1fca2245fc862418223821b763 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/8a548f1fca2245fc862418223821b763 2024-12-08T08:15:33,903 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/recovered.edits/488.seqid, newMaxSeqId=488, maxSeqId=1 2024-12-08T08:15:33,903 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. 2024-12-08T08:15:33,903 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for f652a2fb0fbb79b2d0f57e1ac321a224: Waiting for close lock at 1733645732532Running coprocessor pre-close hooks at 1733645732533 (+1 ms)Disabling compacts and flushes for region at 1733645732533Disabling writes for close at 1733645732533Obtaining lock to block concurrent updates at 1733645732533Preparing flush snapshotting stores in f652a2fb0fbb79b2d0f57e1ac321a224 at 1733645732533Finished memstore snapshotting TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733645732534 (+1 ms)Flushing stores of TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224. at 1733645732535 (+1 ms)Flushing f652a2fb0fbb79b2d0f57e1ac321a224/A: creating writer at 1733645732535Flushing f652a2fb0fbb79b2d0f57e1ac321a224/A: appending metadata at 1733645732539 (+4 ms)Flushing f652a2fb0fbb79b2d0f57e1ac321a224/A: closing flushed file at 1733645732539Flushing f652a2fb0fbb79b2d0f57e1ac321a224/B: creating writer at 1733645732951 (+412 ms)Flushing f652a2fb0fbb79b2d0f57e1ac321a224/B: appending metadata at 1733645732954 (+3 ms)Flushing f652a2fb0fbb79b2d0f57e1ac321a224/B: closing flushed file at 1733645732954Flushing f652a2fb0fbb79b2d0f57e1ac321a224/C: creating writer at 1733645733366 (+412 ms)Flushing f652a2fb0fbb79b2d0f57e1ac321a224/C: appending metadata at 1733645733368 (+2 ms)Flushing f652a2fb0fbb79b2d0f57e1ac321a224/C: closing flushed file at 1733645733368Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f64cfb8: reopening flushed file at 1733645733778 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19994aba: reopening flushed file at 1733645733785 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e0d7976: reopening flushed file at 1733645733789 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for f652a2fb0fbb79b2d0f57e1ac321a224 in 1261ms, sequenceid=485, compaction requested=false at 1733645733794 (+5 ms)Writing region close event to WAL at 1733645733900 (+106 ms)Running coprocessor post-close hooks at 1733645733903 (+3 ms)Closed at 1733645733903 2024-12-08T08:15:33,905 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:33,905 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=f652a2fb0fbb79b2d0f57e1ac321a224, regionState=CLOSED 2024-12-08T08:15:33,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:15:33,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-08T08:15:33,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure f652a2fb0fbb79b2d0f57e1ac321a224, server=422e41b75614,38619,1733645641981 in 1.5270 sec 2024-12-08T08:15:33,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-08T08:15:33,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f652a2fb0fbb79b2d0f57e1ac321a224, UNASSIGN in 1.5310 sec 2024-12-08T08:15:33,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-08T08:15:33,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5330 sec 2024-12-08T08:15:33,911 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645733911"}]},"ts":"1733645733911"} 2024-12-08T08:15:33,913 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T08:15:33,913 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T08:15:33,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5440 sec 2024-12-08T08:15:34,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T08:15:34,521 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:34,521 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-08T08:15:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:34,523 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-08T08:15:34,524 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:34,526 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:34,529 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/recovered.edits] 2024-12-08T08:15:34,534 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/5568e9edfb414229801152e9b0509333 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/5568e9edfb414229801152e9b0509333 2024-12-08T08:15:34,536 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b9b23f7d34b84eb5807fc9d1a54c5be5 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/A/b9b23f7d34b84eb5807fc9d1a54c5be5 2024-12-08T08:15:34,540 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/2a9c939cdd914be589e774137df64276 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/2a9c939cdd914be589e774137df64276 2024-12-08T08:15:34,541 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/de369d703f7f44f28660acffa9561238 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/B/de369d703f7f44f28660acffa9561238 2024-12-08T08:15:34,544 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/4c82c6147df54cc4a4529d2cc982c59a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/4c82c6147df54cc4a4529d2cc982c59a 2024-12-08T08:15:34,545 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/a09a5ca46abb412eb36f695257f6af6e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/C/a09a5ca46abb412eb36f695257f6af6e 2024-12-08T08:15:34,547 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/recovered.edits/488.seqid to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224/recovered.edits/488.seqid 2024-12-08T08:15:34,547 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/f652a2fb0fbb79b2d0f57e1ac321a224 2024-12-08T08:15:34,548 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T08:15:34,549 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:34,552 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T08:15:34,553 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T08:15:34,554 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:34,555 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T08:15:34,555 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733645734555"}]},"ts":"9223372036854775807"} 2024-12-08T08:15:34,556 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T08:15:34,556 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f652a2fb0fbb79b2d0f57e1ac321a224, NAME => 'TestAcidGuarantees,,1733645706078.f652a2fb0fbb79b2d0f57e1ac321a224.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T08:15:34,556 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T08:15:34,556 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733645734556"}]},"ts":"9223372036854775807"} 2024-12-08T08:15:34,558 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T08:15:34,558 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:34,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-12-08T08:15:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-08T08:15:34,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T08:15:34,632 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:34,644 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252 (was 248) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1772320635_22 at /127.0.0.1:49892 [Waiting for operation #289] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2010449533_22 at /127.0.0.1:51048 [Waiting for operation #386] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1772320635_22 at /127.0.0.1:52508 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2010449533_22 at /127.0.0.1:52506 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=417 (was 386) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6526 (was 7460) 2024-12-08T08:15:34,653 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=417, ProcessCount=11, AvailableMemoryMB=6525 2024-12-08T08:15:34,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T08:15:34,654 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:15:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:34,656 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:15:34,656 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:34,656 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-12-08T08:15:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-08T08:15:34,657 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:15:34,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742174_1350 (size=1061) 2024-12-08T08:15:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-08T08:15:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-08T08:15:35,065 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b 2024-12-08T08:15:35,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742175_1351 (size=53) 2024-12-08T08:15:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-08T08:15:35,473 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:15:35,473 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 702b91089e160458511de9d558436b69, disabling compactions & flushes 2024-12-08T08:15:35,473 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:35,474 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:35,474 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. after waiting 0 ms 2024-12-08T08:15:35,474 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:35,474 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:35,474 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 702b91089e160458511de9d558436b69: Waiting for close lock at 1733645735473Disabling compacts and flushes for region at 1733645735473Disabling writes for close at 1733645735474 (+1 ms)Writing region close event to WAL at 1733645735474Closed at 1733645735474 2024-12-08T08:15:35,476 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:15:35,477 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733645735476"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733645735476"}]},"ts":"1733645735476"} 2024-12-08T08:15:35,480 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:15:35,482 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:15:35,482 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645735482"}]},"ts":"1733645735482"} 2024-12-08T08:15:35,483 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T08:15:35,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, ASSIGN}] 2024-12-08T08:15:35,484 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, ASSIGN 2024-12-08T08:15:35,485 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, ASSIGN; state=OFFLINE, location=422e41b75614,38619,1733645641981; forceNewPlan=false, retain=false 2024-12-08T08:15:35,636 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=702b91089e160458511de9d558436b69, regionState=OPENING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:15:35,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, ASSIGN because future has completed 2024-12-08T08:15:35,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:15:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-08T08:15:35,799 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:35,799 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:15:35,800 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 702b91089e160458511de9d558436b69 2024-12-08T08:15:35,800 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:15:35,800 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 702b91089e160458511de9d558436b69 2024-12-08T08:15:35,800 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 702b91089e160458511de9d558436b69 2024-12-08T08:15:35,803 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 702b91089e160458511de9d558436b69 2024-12-08T08:15:35,804 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:15:35,805 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 702b91089e160458511de9d558436b69 columnFamilyName A 2024-12-08T08:15:35,805 DEBUG [StoreOpener-702b91089e160458511de9d558436b69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:35,805 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.HStore(327): Store=702b91089e160458511de9d558436b69/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:15:35,806 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 702b91089e160458511de9d558436b69 2024-12-08T08:15:35,807 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:15:35,807 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 702b91089e160458511de9d558436b69 columnFamilyName B 2024-12-08T08:15:35,807 DEBUG [StoreOpener-702b91089e160458511de9d558436b69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:35,808 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.HStore(327): Store=702b91089e160458511de9d558436b69/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:15:35,808 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 702b91089e160458511de9d558436b69 2024-12-08T08:15:35,810 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:15:35,810 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 702b91089e160458511de9d558436b69 columnFamilyName C 2024-12-08T08:15:35,810 DEBUG [StoreOpener-702b91089e160458511de9d558436b69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:35,811 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.HStore(327): Store=702b91089e160458511de9d558436b69/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:15:35,811 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:35,811 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 702b91089e160458511de9d558436b69 2024-12-08T08:15:35,812 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69 2024-12-08T08:15:35,812 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69 2024-12-08T08:15:35,814 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 702b91089e160458511de9d558436b69 2024-12-08T08:15:35,814 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 702b91089e160458511de9d558436b69 2024-12-08T08:15:35,815 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T08:15:35,817 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 702b91089e160458511de9d558436b69 2024-12-08T08:15:35,818 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:15:35,819 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 702b91089e160458511de9d558436b69; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60085971, jitterRate=-0.10464926064014435}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T08:15:35,819 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 702b91089e160458511de9d558436b69 2024-12-08T08:15:35,819 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 702b91089e160458511de9d558436b69: Running coprocessor pre-open hook at 1733645735800Writing region info on filesystem at 1733645735800Initializing all the Stores at 1733645735802 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645735802Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645735802Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645735802Cleaning up temporary data from old regions at 1733645735814 (+12 ms)Running coprocessor post-open hooks at 1733645735819 (+5 ms)Region opened successfully at 1733645735819 2024-12-08T08:15:35,820 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., pid=91, masterSystemTime=1733645735793 2024-12-08T08:15:35,821 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:35,821 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:35,822 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=702b91089e160458511de9d558436b69, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:15:35,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:15:35,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-08T08:15:35,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 in 183 msec 2024-12-08T08:15:35,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-08T08:15:35,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, ASSIGN in 342 msec 2024-12-08T08:15:35,827 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:15:35,827 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645735827"}]},"ts":"1733645735827"} 2024-12-08T08:15:35,828 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T08:15:35,829 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:15:35,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-12-08T08:15:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-08T08:15:36,811 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:36,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f9102f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:36,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:36,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:36,812 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@629491c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:36,814 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52eb1e4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:36,814 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:36,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:36,816 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T08:15:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 422e41b75614,44803,1733645641278 2024-12-08T08:15:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4f613e60 2024-12-08T08:15:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T08:15:36,818 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T08:15:36,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T08:15:36,819 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:15:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T08:15:36,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742176_1352 (size=1097) 2024-12-08T08:15:37,232 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T08:15:37,232 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T08:15:37,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T08:15:37,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, REOPEN/MOVE}] 2024-12-08T08:15:37,237 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, REOPEN/MOVE 2024-12-08T08:15:37,238 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=702b91089e160458511de9d558436b69, regionState=CLOSING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:15:37,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, REOPEN/MOVE because future has completed 2024-12-08T08:15:37,240 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T08:15:37,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:15:37,392 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,392 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T08:15:37,393 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 702b91089e160458511de9d558436b69, disabling compactions & flushes 2024-12-08T08:15:37,393 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:37,393 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:37,393 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. after waiting 0 ms 2024-12-08T08:15:37,393 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:37,397 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T08:15:37,398 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:37,398 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 702b91089e160458511de9d558436b69: Waiting for close lock at 1733645737393Running coprocessor pre-close hooks at 1733645737393Disabling compacts and flushes for region at 1733645737393Disabling writes for close at 1733645737393Writing region close event to WAL at 1733645737393Running coprocessor post-close hooks at 1733645737398 (+5 ms)Closed at 1733645737398 2024-12-08T08:15:37,398 WARN [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 702b91089e160458511de9d558436b69 to self. 2024-12-08T08:15:37,401 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,402 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=702b91089e160458511de9d558436b69, regionState=CLOSED 2024-12-08T08:15:37,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:15:37,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-08T08:15:37,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 in 165 msec 2024-12-08T08:15:37,408 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, REOPEN/MOVE; state=CLOSED, location=422e41b75614,38619,1733645641981; forceNewPlan=false, retain=true 2024-12-08T08:15:37,559 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=702b91089e160458511de9d558436b69, regionState=OPENING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:15:37,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, REOPEN/MOVE because future has completed 2024-12-08T08:15:37,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:15:37,720 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:37,720 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:15:37,720 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,720 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:15:37,720 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,720 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,722 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,722 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:15:37,723 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 702b91089e160458511de9d558436b69 columnFamilyName A 2024-12-08T08:15:37,724 DEBUG [StoreOpener-702b91089e160458511de9d558436b69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:37,724 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.HStore(327): Store=702b91089e160458511de9d558436b69/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:15:37,724 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,725 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:15:37,725 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 702b91089e160458511de9d558436b69 columnFamilyName B 2024-12-08T08:15:37,725 DEBUG [StoreOpener-702b91089e160458511de9d558436b69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:37,725 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.HStore(327): Store=702b91089e160458511de9d558436b69/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:15:37,725 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,726 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:15:37,726 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 702b91089e160458511de9d558436b69 columnFamilyName C 2024-12-08T08:15:37,726 DEBUG [StoreOpener-702b91089e160458511de9d558436b69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:37,726 INFO [StoreOpener-702b91089e160458511de9d558436b69-1 {}] regionserver.HStore(327): Store=702b91089e160458511de9d558436b69/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:15:37,726 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:37,726 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,727 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69 2024-12-08T08:15:37,728 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69 2024-12-08T08:15:37,728 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,728 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,729 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T08:15:37,730 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,730 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 702b91089e160458511de9d558436b69; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62438498, jitterRate=-0.0695938766002655}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T08:15:37,730 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,731 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 702b91089e160458511de9d558436b69: Running coprocessor pre-open hook at 1733645737720Writing region info on filesystem at 1733645737720Initializing all the Stores at 1733645737721 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645737721Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645737721Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645737721Cleaning up temporary data from old regions at 1733645737728 (+7 ms)Running coprocessor post-open hooks at 1733645737730 (+2 ms)Region opened successfully at 1733645737731 (+1 ms) 2024-12-08T08:15:37,731 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., pid=96, masterSystemTime=1733645737717 2024-12-08T08:15:37,733 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:37,733 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:37,733 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=702b91089e160458511de9d558436b69, regionState=OPEN, openSeqNum=5, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:15:37,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:15:37,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-12-08T08:15:37,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 in 172 msec 2024-12-08T08:15:37,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-08T08:15:37,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, REOPEN/MOVE in 499 msec 2024-12-08T08:15:37,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-08T08:15:37,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-12-08T08:15:37,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 921 msec 2024-12-08T08:15:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-08T08:15:37,742 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:37,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec1958f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,743 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:37,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:37,744 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@548e1aae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:37,745 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@183ac17b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:37,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:37,746 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429fa31a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:37,748 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58c411a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:37,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:37,749 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:37,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fc53c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:37,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:37,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:37,750 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:37,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dde4386, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:37,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:37,752 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:37,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:37,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:37,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a214975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:37,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:37,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:37,753 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:37,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36765321, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:37,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:37,754 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edfce20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:37,756 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:37,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffa265d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:37,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:37,757 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:37,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a3a7cbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:37,759 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6684ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:37,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:37,760 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:37,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:37,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:37,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f1ecffa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:37,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:37,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:37,761 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:37,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@357f1b1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:37,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:37,763 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:37,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:37,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:37,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c711571, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:37,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:37,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:37,765 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755188b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:37,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:37,767 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11f426ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:37,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:37,768 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:37,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8cb8228, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:15:37,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:15:37,770 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:15:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:15:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:15:37,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159ca25a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:15:37,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:15:37,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:37,772 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:15:37,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b813455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:15:37,773 DEBUG [Thread-1610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:37,773 DEBUG [Thread-1601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:37,773 DEBUG [Thread-1616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:37,773 DEBUG [Thread-1618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:37,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:37,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:37,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:37,774 DEBUG [Thread-1614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:37,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:37,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:37,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:37,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:37,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:37,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:37,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:37,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:37,775 DEBUG [Thread-1612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:37,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:37,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:37,775 DEBUG [Thread-1599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:37,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:37,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:37,776 DEBUG [Thread-1603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:37,777 DEBUG [Thread-1605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:37,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:15:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:15:37,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:37,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:15:37,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:15:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:15:37,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:37,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:15:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:15:37,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:15:37,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:15:37,785 DEBUG [Thread-1607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:15:37,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:15:37,786 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:37,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:15:37,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:15:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-08T08:15:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T08:15:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:15:37,792 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:37,793 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:37,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:37,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T08:15:37,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:37,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:37,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:37,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:37,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:37,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:37,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645797867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:37,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645797868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:37,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:37,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645797875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:37,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:37,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120825ed69c5549f43a983c60a137a6fc2c0_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645737799/Put/seqid=0 2024-12-08T08:15:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645797875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:37,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645797876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:37,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742177_1353 (size=14594) 2024-12-08T08:15:37,898 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T08:15:37,901 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120825ed69c5549f43a983c60a137a6fc2c0_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120825ed69c5549f43a983c60a137a6fc2c0_702b91089e160458511de9d558436b69 2024-12-08T08:15:37,902 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/c2fd7aa63bb8410b977cd5f42114992b, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:37,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/c2fd7aa63bb8410b977cd5f42114992b is 175, key is test_row_0/A:col10/1733645737799/Put/seqid=0 2024-12-08T08:15:37,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742178_1354 (size=39549) 2024-12-08T08:15:37,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T08:15:37,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:37,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:37,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:37,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:37,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:37,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645797980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:37,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645797980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:37,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645797990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645797991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645797991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:38,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T08:15:38,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:38,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:38,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:38,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T08:15:38,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645798190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:38,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645798190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:38,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:38,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645798201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645798201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645798201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:38,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T08:15:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:38,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,312 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/c2fd7aa63bb8410b977cd5f42114992b 2024-12-08T08:15:38,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/3ac37d2cc3fa4c40ae79a9bbb1833e8a is 50, key is test_row_0/B:col10/1733645737799/Put/seqid=0 2024-12-08T08:15:38,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742179_1355 (size=12001) 2024-12-08T08:15:38,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/3ac37d2cc3fa4c40ae79a9bbb1833e8a 2024-12-08T08:15:38,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/90c13e228efc4aafa66e6fb4aa713dbe is 50, key is test_row_0/C:col10/1733645737799/Put/seqid=0 2024-12-08T08:15:38,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742180_1356 (size=12001) 2024-12-08T08:15:38,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T08:15:38,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:38,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:38,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:38,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T08:15:38,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645798500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645798500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:38,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645798511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:38,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645798511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:38,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645798511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:38,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T08:15:38,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:38,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:38,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:38,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T08:15:38,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:38,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:38,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:38,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:38,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/90c13e228efc4aafa66e6fb4aa713dbe 2024-12-08T08:15:38,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/c2fd7aa63bb8410b977cd5f42114992b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/c2fd7aa63bb8410b977cd5f42114992b 2024-12-08T08:15:38,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/c2fd7aa63bb8410b977cd5f42114992b, entries=200, sequenceid=17, filesize=38.6 K 2024-12-08T08:15:38,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/3ac37d2cc3fa4c40ae79a9bbb1833e8a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3ac37d2cc3fa4c40ae79a9bbb1833e8a 2024-12-08T08:15:38,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3ac37d2cc3fa4c40ae79a9bbb1833e8a, entries=150, sequenceid=17, filesize=11.7 K 2024-12-08T08:15:38,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/90c13e228efc4aafa66e6fb4aa713dbe as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/90c13e228efc4aafa66e6fb4aa713dbe 2024-12-08T08:15:38,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/90c13e228efc4aafa66e6fb4aa713dbe, entries=150, sequenceid=17, filesize=11.7 K 2024-12-08T08:15:38,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 702b91089e160458511de9d558436b69 in 993ms, sequenceid=17, compaction requested=false 2024-12-08T08:15:38,821 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T08:15:38,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:38,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T08:15:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:38,871 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T08:15:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:38,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208cfe0dd7d21f44ee3a7f36c86a2101c98_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645737875/Put/seqid=0 2024-12-08T08:15:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T08:15:38,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742181_1357 (size=12154) 2024-12-08T08:15:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:38,950 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208cfe0dd7d21f44ee3a7f36c86a2101c98_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208cfe0dd7d21f44ee3a7f36c86a2101c98_702b91089e160458511de9d558436b69 2024-12-08T08:15:38,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/69ec92ee857c475891bb278fa399f12f, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:38,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/69ec92ee857c475891bb278fa399f12f is 175, key is test_row_0/A:col10/1733645737875/Put/seqid=0 2024-12-08T08:15:38,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742182_1358 (size=30955) 2024-12-08T08:15:38,991 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/69ec92ee857c475891bb278fa399f12f 2024-12-08T08:15:39,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/25554c7f61d84f52ba3e521f42d0c064 is 50, key is test_row_0/B:col10/1733645737875/Put/seqid=0 2024-12-08T08:15:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:39,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:39,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742183_1359 (size=12001) 2024-12-08T08:15:39,022 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/25554c7f61d84f52ba3e521f42d0c064 2024-12-08T08:15:39,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/abd6b213aa714cee9179af26f39a72f3 is 50, key is test_row_0/C:col10/1733645737875/Put/seqid=0 2024-12-08T08:15:39,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645799034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645799034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645799035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645799042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645799043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742184_1360 (size=12001) 2024-12-08T08:15:39,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645799150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645799151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645799151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645799151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645799161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645799360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645799360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645799361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645799361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645799371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,469 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T08:15:39,549 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/abd6b213aa714cee9179af26f39a72f3 2024-12-08T08:15:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/69ec92ee857c475891bb278fa399f12f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/69ec92ee857c475891bb278fa399f12f 2024-12-08T08:15:39,560 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/69ec92ee857c475891bb278fa399f12f, entries=150, sequenceid=36, filesize=30.2 K 2024-12-08T08:15:39,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/25554c7f61d84f52ba3e521f42d0c064 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/25554c7f61d84f52ba3e521f42d0c064 2024-12-08T08:15:39,566 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/25554c7f61d84f52ba3e521f42d0c064, entries=150, sequenceid=36, filesize=11.7 K 2024-12-08T08:15:39,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/abd6b213aa714cee9179af26f39a72f3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/abd6b213aa714cee9179af26f39a72f3 2024-12-08T08:15:39,571 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/abd6b213aa714cee9179af26f39a72f3, entries=150, sequenceid=36, filesize=11.7 K 2024-12-08T08:15:39,572 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 702b91089e160458511de9d558436b69 in 701ms, sequenceid=36, compaction requested=false 2024-12-08T08:15:39,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:39,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:39,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-08T08:15:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-08T08:15:39,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-08T08:15:39,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7810 sec 2024-12-08T08:15:39,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.7900 sec 2024-12-08T08:15:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:39,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T08:15:39,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:39,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:39,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:39,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:39,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:39,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:39,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208550ee05e6f944efc8af898527e1144a8_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645739673/Put/seqid=0 2024-12-08T08:15:39,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742185_1361 (size=14594) 2024-12-08T08:15:39,690 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:39,694 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208550ee05e6f944efc8af898527e1144a8_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208550ee05e6f944efc8af898527e1144a8_702b91089e160458511de9d558436b69 2024-12-08T08:15:39,694 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/a876237665464f3089687f8ea93968fa, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:39,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/a876237665464f3089687f8ea93968fa is 175, key is test_row_0/A:col10/1733645739673/Put/seqid=0 2024-12-08T08:15:39,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742186_1362 (size=39549) 2024-12-08T08:15:39,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645799697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645799703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645799703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645799704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645799705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645799811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645799821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645799821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645799821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645799821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:39,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:39,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T08:15:39,940 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:39,941 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-08T08:15:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T08:15:39,942 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:39,943 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:39,943 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:40,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645800020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645800030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645800030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645800030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645800030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T08:15:40,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T08:15:40,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:40,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,100 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/a876237665464f3089687f8ea93968fa 2024-12-08T08:15:40,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/ae4e55bdd8bf44bd80158efdd3f792fa is 50, key is test_row_0/B:col10/1733645739673/Put/seqid=0 2024-12-08T08:15:40,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742187_1363 (size=12001) 2024-12-08T08:15:40,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T08:15:40,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:40,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T08:15:40,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645800331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645800340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645800340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645800340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645800341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T08:15:40,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:40,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/ae4e55bdd8bf44bd80158efdd3f792fa 2024-12-08T08:15:40,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/08c87666999047cf933eddd7b1884c77 is 50, key is test_row_0/C:col10/1733645739673/Put/seqid=0 2024-12-08T08:15:40,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742188_1364 (size=12001) 2024-12-08T08:15:40,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T08:15:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T08:15:40,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T08:15:40,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:40,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645800850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645800850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645800850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T08:15:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:40,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645800861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645800861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:40,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/08c87666999047cf933eddd7b1884c77 2024-12-08T08:15:40,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/a876237665464f3089687f8ea93968fa as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/a876237665464f3089687f8ea93968fa 2024-12-08T08:15:40,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/a876237665464f3089687f8ea93968fa, entries=200, sequenceid=50, filesize=38.6 K 2024-12-08T08:15:40,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/ae4e55bdd8bf44bd80158efdd3f792fa as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/ae4e55bdd8bf44bd80158efdd3f792fa 2024-12-08T08:15:40,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/ae4e55bdd8bf44bd80158efdd3f792fa, entries=150, sequenceid=50, filesize=11.7 K 2024-12-08T08:15:40,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/08c87666999047cf933eddd7b1884c77 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/08c87666999047cf933eddd7b1884c77 2024-12-08T08:15:40,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/08c87666999047cf933eddd7b1884c77, entries=150, sequenceid=50, filesize=11.7 K 2024-12-08T08:15:40,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 702b91089e160458511de9d558436b69 in 1261ms, sequenceid=50, compaction requested=true 2024-12-08T08:15:40,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:40,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:40,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:40,935 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:40,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:40,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:40,935 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:40,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:40,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:40,936 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:40,936 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:40,936 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/A is initiating minor compaction (all files) 2024-12-08T08:15:40,936 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/B is initiating minor compaction (all files) 2024-12-08T08:15:40,936 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/B in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,936 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/A in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,936 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/c2fd7aa63bb8410b977cd5f42114992b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/69ec92ee857c475891bb278fa399f12f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/a876237665464f3089687f8ea93968fa] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=107.5 K 2024-12-08T08:15:40,936 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3ac37d2cc3fa4c40ae79a9bbb1833e8a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/25554c7f61d84f52ba3e521f42d0c064, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/ae4e55bdd8bf44bd80158efdd3f792fa] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=35.2 K 2024-12-08T08:15:40,936 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:40,936 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/c2fd7aa63bb8410b977cd5f42114992b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/69ec92ee857c475891bb278fa399f12f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/a876237665464f3089687f8ea93968fa] 2024-12-08T08:15:40,936 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ac37d2cc3fa4c40ae79a9bbb1833e8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733645737784 2024-12-08T08:15:40,936 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2fd7aa63bb8410b977cd5f42114992b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733645737784 2024-12-08T08:15:40,937 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69ec92ee857c475891bb278fa399f12f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733645737866 2024-12-08T08:15:40,937 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 25554c7f61d84f52ba3e521f42d0c064, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733645737866 2024-12-08T08:15:40,937 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting a876237665464f3089687f8ea93968fa, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733645739025 2024-12-08T08:15:40,937 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting ae4e55bdd8bf44bd80158efdd3f792fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733645739025 2024-12-08T08:15:40,941 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:40,943 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#B#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:40,943 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/534cd88268cc44caa4764ff749231827 is 50, key is test_row_0/B:col10/1733645739673/Put/seqid=0 2024-12-08T08:15:40,945 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412080cacba0bc7f144b7b7109c352b26b131_702b91089e160458511de9d558436b69 store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:40,946 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412080cacba0bc7f144b7b7109c352b26b131_702b91089e160458511de9d558436b69, store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:40,946 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080cacba0bc7f144b7b7109c352b26b131_702b91089e160458511de9d558436b69 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:40,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742189_1365 (size=12104) 2024-12-08T08:15:40,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742190_1366 (size=4469) 2024-12-08T08:15:41,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T08:15:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:41,011 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T08:15:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:41,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:41,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083d30e04b523c4672b5da99a9486d615c_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645739704/Put/seqid=0 2024-12-08T08:15:41,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742191_1367 (size=12154) 2024-12-08T08:15:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T08:15:41,359 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#A#compaction#312 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:41,360 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/781a933695934cd98ce2a6fc443ce009 is 175, key is test_row_0/A:col10/1733645739673/Put/seqid=0 2024-12-08T08:15:41,360 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/534cd88268cc44caa4764ff749231827 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/534cd88268cc44caa4764ff749231827 2024-12-08T08:15:41,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742192_1368 (size=31058) 2024-12-08T08:15:41,366 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 702b91089e160458511de9d558436b69/B of 702b91089e160458511de9d558436b69 into 534cd88268cc44caa4764ff749231827(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:41,366 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:41,366 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/B, priority=13, startTime=1733645740935; duration=0sec 2024-12-08T08:15:41,366 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:41,366 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:B 2024-12-08T08:15:41,366 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:41,367 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:41,367 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/C is initiating minor compaction (all files) 2024-12-08T08:15:41,367 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/C in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:41,367 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/90c13e228efc4aafa66e6fb4aa713dbe, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/abd6b213aa714cee9179af26f39a72f3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/08c87666999047cf933eddd7b1884c77] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=35.2 K 2024-12-08T08:15:41,368 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 90c13e228efc4aafa66e6fb4aa713dbe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733645737784 2024-12-08T08:15:41,368 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting abd6b213aa714cee9179af26f39a72f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733645737866 2024-12-08T08:15:41,368 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 08c87666999047cf933eddd7b1884c77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733645739025 2024-12-08T08:15:41,371 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/781a933695934cd98ce2a6fc443ce009 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/781a933695934cd98ce2a6fc443ce009 2024-12-08T08:15:41,375 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 702b91089e160458511de9d558436b69/A of 702b91089e160458511de9d558436b69 into 781a933695934cd98ce2a6fc443ce009(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:41,375 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:41,375 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/A, priority=13, startTime=1733645740935; duration=0sec 2024-12-08T08:15:41,375 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:41,375 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:A 2024-12-08T08:15:41,375 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#C#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:41,376 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/2b0d27e1663a4a74ab18353fa50b6538 is 50, key is test_row_0/C:col10/1733645739673/Put/seqid=0 2024-12-08T08:15:41,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742193_1369 (size=12104) 2024-12-08T08:15:41,383 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/2b0d27e1663a4a74ab18353fa50b6538 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/2b0d27e1663a4a74ab18353fa50b6538 2024-12-08T08:15:41,389 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 702b91089e160458511de9d558436b69/C of 702b91089e160458511de9d558436b69 into 2b0d27e1663a4a74ab18353fa50b6538(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:41,389 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:41,389 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/C, priority=13, startTime=1733645740935; duration=0sec 2024-12-08T08:15:41,389 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:41,389 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:C 2024-12-08T08:15:41,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:41,458 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083d30e04b523c4672b5da99a9486d615c_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083d30e04b523c4672b5da99a9486d615c_702b91089e160458511de9d558436b69 2024-12-08T08:15:41,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/8efa10d408c842ba8e19155f5574906b, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:41,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/8efa10d408c842ba8e19155f5574906b is 175, key is test_row_0/A:col10/1733645739704/Put/seqid=0 2024-12-08T08:15:41,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742194_1370 (size=30955) 2024-12-08T08:15:41,467 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/8efa10d408c842ba8e19155f5574906b 2024-12-08T08:15:41,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/88e12fbb11664b68878266b31d73571d is 50, key is test_row_0/B:col10/1733645739704/Put/seqid=0 2024-12-08T08:15:41,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742195_1371 (size=12001) 2024-12-08T08:15:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:41,890 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/88e12fbb11664b68878266b31d73571d 2024-12-08T08:15:41,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645801881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:41,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645801885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:41,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645801891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:41,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/141cca72f4dd48e98f383091eb331032 is 50, key is test_row_0/C:col10/1733645739704/Put/seqid=0 2024-12-08T08:15:41,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645801893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:41,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645801893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:41,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742196_1372 (size=12001) 2024-12-08T08:15:42,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645802000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645802001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645802001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645802011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645802011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T08:15:42,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645802211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645802211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645802211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645802220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645802220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,306 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/141cca72f4dd48e98f383091eb331032 2024-12-08T08:15:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/8efa10d408c842ba8e19155f5574906b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8efa10d408c842ba8e19155f5574906b 2024-12-08T08:15:42,323 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8efa10d408c842ba8e19155f5574906b, entries=150, sequenceid=68, filesize=30.2 K 2024-12-08T08:15:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/88e12fbb11664b68878266b31d73571d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/88e12fbb11664b68878266b31d73571d 2024-12-08T08:15:42,326 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/88e12fbb11664b68878266b31d73571d, entries=150, sequenceid=68, filesize=11.7 K 2024-12-08T08:15:42,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/141cca72f4dd48e98f383091eb331032 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/141cca72f4dd48e98f383091eb331032 2024-12-08T08:15:42,334 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/141cca72f4dd48e98f383091eb331032, entries=150, sequenceid=68, filesize=11.7 K 2024-12-08T08:15:42,334 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 702b91089e160458511de9d558436b69 in 1323ms, sequenceid=68, compaction requested=false 2024-12-08T08:15:42,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:42,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:42,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-08T08:15:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-08T08:15:42,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-08T08:15:42,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3930 sec 2024-12-08T08:15:42,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.4000 sec 2024-12-08T08:15:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:42,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T08:15:42,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:42,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:42,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:42,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:42,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:42,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:42,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087bb5deef698d421e9180143a27b178b0_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645742525/Put/seqid=0 2024-12-08T08:15:42,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742197_1373 (size=14594) 2024-12-08T08:15:42,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645802547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645802548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645802550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645802551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645802552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645802660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645802660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645802660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645802671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645802671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645802870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645802870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645802871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645802881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645802881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:42,943 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:42,946 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087bb5deef698d421e9180143a27b178b0_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087bb5deef698d421e9180143a27b178b0_702b91089e160458511de9d558436b69 2024-12-08T08:15:42,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/dc212aefab4a443eacd0f2e12c96d0a0, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:42,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/dc212aefab4a443eacd0f2e12c96d0a0 is 175, key is test_row_0/A:col10/1733645742525/Put/seqid=0 2024-12-08T08:15:42,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742198_1374 (size=39549) 2024-12-08T08:15:43,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645803180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645803180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645803180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:43,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645803191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:43,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:43,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645803191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:43,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:43,350 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/dc212aefab4a443eacd0f2e12c96d0a0 2024-12-08T08:15:43,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/aa52c08b730a4365a43c5e70229dc0a2 is 50, key is test_row_0/B:col10/1733645742525/Put/seqid=0 2024-12-08T08:15:43,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742199_1375 (size=12001) 2024-12-08T08:15:43,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/aa52c08b730a4365a43c5e70229dc0a2 2024-12-08T08:15:43,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/86cbe3ea52cd43928e74c28a214cd570 is 50, key is test_row_0/C:col10/1733645742525/Put/seqid=0 2024-12-08T08:15:43,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742200_1376 (size=12001) 2024-12-08T08:15:43,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645803691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:43,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645803691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:43,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645803700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:43,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645803701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645803701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:43,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/86cbe3ea52cd43928e74c28a214cd570 2024-12-08T08:15:43,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/dc212aefab4a443eacd0f2e12c96d0a0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/dc212aefab4a443eacd0f2e12c96d0a0 2024-12-08T08:15:43,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/dc212aefab4a443eacd0f2e12c96d0a0, entries=200, sequenceid=86, filesize=38.6 K 2024-12-08T08:15:43,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/aa52c08b730a4365a43c5e70229dc0a2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/aa52c08b730a4365a43c5e70229dc0a2 2024-12-08T08:15:43,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/aa52c08b730a4365a43c5e70229dc0a2, entries=150, sequenceid=86, filesize=11.7 K 2024-12-08T08:15:43,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/86cbe3ea52cd43928e74c28a214cd570 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/86cbe3ea52cd43928e74c28a214cd570 2024-12-08T08:15:43,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/86cbe3ea52cd43928e74c28a214cd570, entries=150, sequenceid=86, filesize=11.7 K 2024-12-08T08:15:43,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 702b91089e160458511de9d558436b69 in 1264ms, sequenceid=86, compaction requested=true 2024-12-08T08:15:43,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:43,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:43,791 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:43,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:43,791 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:43,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:43,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:43,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:43,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:43,792 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:43,792 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/A is initiating minor compaction (all files) 2024-12-08T08:15:43,792 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/A in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:43,792 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/781a933695934cd98ce2a6fc443ce009, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8efa10d408c842ba8e19155f5574906b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/dc212aefab4a443eacd0f2e12c96d0a0] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=99.2 K 2024-12-08T08:15:43,792 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:43,792 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/781a933695934cd98ce2a6fc443ce009, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8efa10d408c842ba8e19155f5574906b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/dc212aefab4a443eacd0f2e12c96d0a0] 2024-12-08T08:15:43,793 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:43,793 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 781a933695934cd98ce2a6fc443ce009, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733645739025 2024-12-08T08:15:43,793 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/B is initiating minor compaction (all files) 2024-12-08T08:15:43,793 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/B in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:43,793 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/534cd88268cc44caa4764ff749231827, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/88e12fbb11664b68878266b31d73571d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/aa52c08b730a4365a43c5e70229dc0a2] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=35.3 K 2024-12-08T08:15:43,793 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8efa10d408c842ba8e19155f5574906b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733645739702 2024-12-08T08:15:43,793 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 534cd88268cc44caa4764ff749231827, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733645739025 2024-12-08T08:15:43,793 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc212aefab4a443eacd0f2e12c96d0a0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733645741889 2024-12-08T08:15:43,793 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 88e12fbb11664b68878266b31d73571d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733645739702 2024-12-08T08:15:43,794 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting aa52c08b730a4365a43c5e70229dc0a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733645741889 2024-12-08T08:15:43,800 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#B#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:43,800 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/2c47114679c042d8916a27e7eaa82578 is 50, key is test_row_0/B:col10/1733645742525/Put/seqid=0 2024-12-08T08:15:43,800 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:43,802 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412082512bcf7822541b89f335c38e6f3a243_702b91089e160458511de9d558436b69 store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:43,803 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412082512bcf7822541b89f335c38e6f3a243_702b91089e160458511de9d558436b69, store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:43,803 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082512bcf7822541b89f335c38e6f3a243_702b91089e160458511de9d558436b69 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:43,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742201_1377 (size=12207) 2024-12-08T08:15:43,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742202_1378 (size=4469) 2024-12-08T08:15:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T08:15:44,100 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:44,101 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-08T08:15:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T08:15:44,102 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:44,103 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:44,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:44,209 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/2c47114679c042d8916a27e7eaa82578 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/2c47114679c042d8916a27e7eaa82578 2024-12-08T08:15:44,209 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#A#compaction#322 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:44,210 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/f380eb2dc1d8403587fc4c7bd4bd558b is 175, key is test_row_0/A:col10/1733645742525/Put/seqid=0 2024-12-08T08:15:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T08:15:44,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742203_1379 (size=31161) 2024-12-08T08:15:44,224 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 702b91089e160458511de9d558436b69/B of 702b91089e160458511de9d558436b69 into 2c47114679c042d8916a27e7eaa82578(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:44,224 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:44,224 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/B, priority=13, startTime=1733645743791; duration=0sec 2024-12-08T08:15:44,224 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:44,224 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:B 2024-12-08T08:15:44,224 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:44,227 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:44,227 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/C is initiating minor compaction (all files) 2024-12-08T08:15:44,227 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/C in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:44,227 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/2b0d27e1663a4a74ab18353fa50b6538, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/141cca72f4dd48e98f383091eb331032, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/86cbe3ea52cd43928e74c28a214cd570] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=35.3 K 2024-12-08T08:15:44,227 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b0d27e1663a4a74ab18353fa50b6538, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733645739025 2024-12-08T08:15:44,227 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 141cca72f4dd48e98f383091eb331032, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733645739702 2024-12-08T08:15:44,228 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 86cbe3ea52cd43928e74c28a214cd570, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733645741889 2024-12-08T08:15:44,234 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#C#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:44,235 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/cd2f5b43c318461bac2ed4dd2de24fd8 is 50, key is test_row_0/C:col10/1733645742525/Put/seqid=0 2024-12-08T08:15:44,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742204_1380 (size=12207) 2024-12-08T08:15:44,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-08T08:15:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:44,256 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T08:15:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:44,271 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/cd2f5b43c318461bac2ed4dd2de24fd8 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/cd2f5b43c318461bac2ed4dd2de24fd8 2024-12-08T08:15:44,275 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 702b91089e160458511de9d558436b69/C of 702b91089e160458511de9d558436b69 into cd2f5b43c318461bac2ed4dd2de24fd8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:44,275 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:44,275 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/C, priority=13, startTime=1733645743791; duration=0sec 2024-12-08T08:15:44,275 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:44,275 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:C 2024-12-08T08:15:44,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088f098df96d4a487c9e891d469d7bfcac_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645742550/Put/seqid=0 2024-12-08T08:15:44,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742205_1381 (size=12154) 2024-12-08T08:15:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T08:15:44,625 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/f380eb2dc1d8403587fc4c7bd4bd558b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f380eb2dc1d8403587fc4c7bd4bd558b 2024-12-08T08:15:44,632 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 702b91089e160458511de9d558436b69/A of 702b91089e160458511de9d558436b69 into f380eb2dc1d8403587fc4c7bd4bd558b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:44,632 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:44,632 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/A, priority=13, startTime=1733645743791; duration=0sec 2024-12-08T08:15:44,632 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:44,632 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:A 2024-12-08T08:15:44,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:44,695 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088f098df96d4a487c9e891d469d7bfcac_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088f098df96d4a487c9e891d469d7bfcac_702b91089e160458511de9d558436b69 2024-12-08T08:15:44,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/5799b13559f740b0bf2cbe4620c9cd51, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:44,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/5799b13559f740b0bf2cbe4620c9cd51 is 175, key is test_row_0/A:col10/1733645742550/Put/seqid=0 2024-12-08T08:15:44,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742206_1382 (size=30955) 2024-12-08T08:15:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:44,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:44,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:44,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645804721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:44,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:44,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645804725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:44,727 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_0' in 702b91089e160458511de9d558436b69 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:44,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:44,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:44,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645804725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:44,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T08:15:44,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645804726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:44,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:44,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645804727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:44,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:44,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645804831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:44,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:44,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645804831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:44,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645804841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:44,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645804841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:45,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645805041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645805041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:45,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645805051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:45,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:45,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645805051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:45,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:45,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:45,100 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/5799b13559f740b0bf2cbe4620c9cd51 2024-12-08T08:15:45,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/0e06da802d814bc5aa2a62a38da89d82 is 50, key is test_row_0/B:col10/1733645742550/Put/seqid=0 2024-12-08T08:15:45,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742207_1383 (size=12001) 2024-12-08T08:15:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T08:15:45,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:45,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645805350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645805350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:45,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:45,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:45,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645805361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:45,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:45,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645805361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:45,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:45,510 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/0e06da802d814bc5aa2a62a38da89d82 2024-12-08T08:15:45,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/70a79e46fe8a4a7fbcaee93128b049d7 is 50, key is test_row_0/C:col10/1733645742550/Put/seqid=0 2024-12-08T08:15:45,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742208_1384 (size=12001) 2024-12-08T08:15:45,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645805860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:45,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:45,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645805871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:45,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645805871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645805871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:45,925 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/70a79e46fe8a4a7fbcaee93128b049d7 2024-12-08T08:15:45,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/5799b13559f740b0bf2cbe4620c9cd51 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/5799b13559f740b0bf2cbe4620c9cd51 2024-12-08T08:15:45,931 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/5799b13559f740b0bf2cbe4620c9cd51, entries=150, sequenceid=104, filesize=30.2 K 2024-12-08T08:15:45,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/0e06da802d814bc5aa2a62a38da89d82 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/0e06da802d814bc5aa2a62a38da89d82 2024-12-08T08:15:45,935 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/0e06da802d814bc5aa2a62a38da89d82, entries=150, sequenceid=104, filesize=11.7 K 2024-12-08T08:15:45,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/70a79e46fe8a4a7fbcaee93128b049d7 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/70a79e46fe8a4a7fbcaee93128b049d7 2024-12-08T08:15:45,939 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/70a79e46fe8a4a7fbcaee93128b049d7, entries=150, sequenceid=104, filesize=11.7 K 2024-12-08T08:15:45,940 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 702b91089e160458511de9d558436b69 in 1684ms, sequenceid=104, compaction requested=false 2024-12-08T08:15:45,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:45,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:45,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-08T08:15:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-08T08:15:45,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-08T08:15:45,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8380 sec 2024-12-08T08:15:45,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.8420 sec 2024-12-08T08:15:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T08:15:46,251 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:46,251 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-08T08:15:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T08:15:46,253 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:46,253 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:46,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T08:15:46,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T08:15:46,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:46,406 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T08:15:46,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:46,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:46,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:46,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:46,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:46,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:46,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208309bca6d4e5749c681c76f0e78f45b2a_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645744715/Put/seqid=0 2024-12-08T08:15:46,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742209_1385 (size=12154) 2024-12-08T08:15:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T08:15:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:46,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:46,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:46,831 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208309bca6d4e5749c681c76f0e78f45b2a_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208309bca6d4e5749c681c76f0e78f45b2a_702b91089e160458511de9d558436b69 2024-12-08T08:15:46,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/6a41d1e0335144e9a2a3dbf66c94478c, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:46,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/6a41d1e0335144e9a2a3dbf66c94478c is 175, key is test_row_0/A:col10/1733645744715/Put/seqid=0 2024-12-08T08:15:46,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742210_1386 (size=30955) 2024-12-08T08:15:46,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645806841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T08:15:46,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645806881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:46,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645806881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:46,887 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 702b91089e160458511de9d558436b69 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645806881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:46,888 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 702b91089e160458511de9d558436b69 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645806881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:46,888 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 702b91089e160458511de9d558436b69 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:46,888 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 702b91089e160458511de9d558436b69 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:46,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645806961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:47,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645807171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:47,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:47,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:47,242 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/6a41d1e0335144e9a2a3dbf66c94478c 2024-12-08T08:15:47,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/3cee63bfcd9845c78a7d10c60c23e275 is 50, key is test_row_0/B:col10/1733645744715/Put/seqid=0 2024-12-08T08:15:47,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742211_1387 (size=12001) 2024-12-08T08:15:47,270 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/3cee63bfcd9845c78a7d10c60c23e275 2024-12-08T08:15:47,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/d5261a7fd4ee436fb2206bf65d966fe3 is 50, key is test_row_0/C:col10/1733645744715/Put/seqid=0 2024-12-08T08:15:47,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742212_1388 (size=12001) 2024-12-08T08:15:47,286 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/d5261a7fd4ee436fb2206bf65d966fe3 2024-12-08T08:15:47,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/6a41d1e0335144e9a2a3dbf66c94478c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6a41d1e0335144e9a2a3dbf66c94478c 2024-12-08T08:15:47,296 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6a41d1e0335144e9a2a3dbf66c94478c, entries=150, sequenceid=121, filesize=30.2 K 2024-12-08T08:15:47,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/3cee63bfcd9845c78a7d10c60c23e275 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3cee63bfcd9845c78a7d10c60c23e275 2024-12-08T08:15:47,301 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3cee63bfcd9845c78a7d10c60c23e275, entries=150, sequenceid=121, filesize=11.7 K 2024-12-08T08:15:47,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/d5261a7fd4ee436fb2206bf65d966fe3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/d5261a7fd4ee436fb2206bf65d966fe3 2024-12-08T08:15:47,304 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/d5261a7fd4ee436fb2206bf65d966fe3, entries=150, sequenceid=121, filesize=11.7 K 2024-12-08T08:15:47,306 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 702b91089e160458511de9d558436b69 in 901ms, sequenceid=121, compaction requested=true 2024-12-08T08:15:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:47,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-08T08:15:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-08T08:15:47,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-08T08:15:47,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0560 sec 2024-12-08T08:15:47,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.0600 sec 2024-12-08T08:15:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T08:15:47,391 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:47,391 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-08T08:15:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T08:15:47,392 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:47,393 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:47,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:47,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T08:15:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:47,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089ea2747155274c59b145df0644f0e35f_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645747481/Put/seqid=0 2024-12-08T08:15:47,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742213_1389 (size=14794) 2024-12-08T08:15:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T08:15:47,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645807539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:47,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T08:15:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:47,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:47,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:47,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:47,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:47,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645807650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:47,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T08:15:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:47,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T08:15:47,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T08:15:47,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:47,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:47,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:47,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:47,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:47,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645807860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:47,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:47,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:47,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:47,893 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:47,906 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089ea2747155274c59b145df0644f0e35f_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089ea2747155274c59b145df0644f0e35f_702b91089e160458511de9d558436b69 2024-12-08T08:15:47,907 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/ee8a75f2e84a4980b891c3b59e713be0, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:47,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/ee8a75f2e84a4980b891c3b59e713be0 is 175, key is test_row_0/A:col10/1733645747481/Put/seqid=0 2024-12-08T08:15:47,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742214_1390 (size=39749) 2024-12-08T08:15:48,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T08:15:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T08:15:48,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T08:15:48,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:48,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645808170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:48,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T08:15:48,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:48,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,322 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/ee8a75f2e84a4980b891c3b59e713be0 2024-12-08T08:15:48,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/e017d72fbaa849fbb7d813a21c5df195 is 50, key is test_row_0/B:col10/1733645747481/Put/seqid=0 2024-12-08T08:15:48,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742215_1391 (size=12151) 2024-12-08T08:15:48,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/e017d72fbaa849fbb7d813a21c5df195 2024-12-08T08:15:48,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/414bf55d101748729dd6ef8e4b5a47ad is 50, key is test_row_0/C:col10/1733645747481/Put/seqid=0 2024-12-08T08:15:48,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742216_1392 (size=12151) 2024-12-08T08:15:48,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T08:15:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T08:15:48,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T08:15:48,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:48,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645808680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:48,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/414bf55d101748729dd6ef8e4b5a47ad 2024-12-08T08:15:48,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/ee8a75f2e84a4980b891c3b59e713be0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/ee8a75f2e84a4980b891c3b59e713be0 2024-12-08T08:15:48,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/ee8a75f2e84a4980b891c3b59e713be0, entries=200, sequenceid=139, filesize=38.8 K 2024-12-08T08:15:48,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/e017d72fbaa849fbb7d813a21c5df195 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/e017d72fbaa849fbb7d813a21c5df195 2024-12-08T08:15:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/e017d72fbaa849fbb7d813a21c5df195, entries=150, sequenceid=139, filesize=11.9 K 2024-12-08T08:15:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/414bf55d101748729dd6ef8e4b5a47ad as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/414bf55d101748729dd6ef8e4b5a47ad 2024-12-08T08:15:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/414bf55d101748729dd6ef8e4b5a47ad, entries=150, sequenceid=139, filesize=11.9 K 2024-12-08T08:15:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 702b91089e160458511de9d558436b69 in 1277ms, sequenceid=139, compaction requested=true 2024-12-08T08:15:48,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:48,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:48,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:48,759 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:15:48,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:48,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:48,759 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:15:48,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:48,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,760 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:15:48,760 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/A is initiating minor compaction (all files) 2024-12-08T08:15:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,760 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/A in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,760 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f380eb2dc1d8403587fc4c7bd4bd558b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/5799b13559f740b0bf2cbe4620c9cd51, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6a41d1e0335144e9a2a3dbf66c94478c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/ee8a75f2e84a4980b891c3b59e713be0] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=129.7 K 2024-12-08T08:15:48,760 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,760 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f380eb2dc1d8403587fc4c7bd4bd558b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/5799b13559f740b0bf2cbe4620c9cd51, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6a41d1e0335144e9a2a3dbf66c94478c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/ee8a75f2e84a4980b891c3b59e713be0] 2024-12-08T08:15:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,761 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:15:48,761 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/B is initiating minor compaction (all files) 2024-12-08T08:15:48,761 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting f380eb2dc1d8403587fc4c7bd4bd558b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733645741889 2024-12-08T08:15:48,761 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/B in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,761 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/2c47114679c042d8916a27e7eaa82578, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/0e06da802d814bc5aa2a62a38da89d82, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3cee63bfcd9845c78a7d10c60c23e275, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/e017d72fbaa849fbb7d813a21c5df195] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=47.2 K 2024-12-08T08:15:48,761 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5799b13559f740b0bf2cbe4620c9cd51, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733645742547 2024-12-08T08:15:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,761 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c47114679c042d8916a27e7eaa82578, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733645741889 2024-12-08T08:15:48,761 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a41d1e0335144e9a2a3dbf66c94478c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733645744715 2024-12-08T08:15:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,761 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e06da802d814bc5aa2a62a38da89d82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733645742547 2024-12-08T08:15:48,762 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee8a75f2e84a4980b891c3b59e713be0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733645746824 2024-12-08T08:15:48,762 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cee63bfcd9845c78a7d10c60c23e275, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733645744715 2024-12-08T08:15:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,762 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting e017d72fbaa849fbb7d813a21c5df195, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733645746829 2024-12-08T08:15:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T08:15:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,765 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T08:15:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:48,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:48,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:48,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:48,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:48,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,767 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,769 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208243dc1f9fcca47afb855660f2d000586_702b91089e160458511de9d558436b69 store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,769 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,770 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/a63a85ca96814e78a7dc14815c6290fa is 50, key is test_row_0/B:col10/1733645747481/Put/seqid=0 2024-12-08T08:15:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120870157324ba194ab9b4e48bf7bad11f50_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645747538/Put/seqid=0 2024-12-08T08:15:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,771 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208243dc1f9fcca47afb855660f2d000586_702b91089e160458511de9d558436b69, store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,771 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208243dc1f9fcca47afb855660f2d000586_702b91089e160458511de9d558436b69 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742217_1393 (size=12493) 2024-12-08T08:15:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742218_1394 (size=4469) 2024-12-08T08:15:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,785 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#A#compaction#333 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,786 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/f340a390c5554798b29f132e44f71994 is 175, key is test_row_0/A:col10/1733645747481/Put/seqid=0 2024-12-08T08:15:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742220_1396 (size=31447) 2024-12-08T08:15:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742219_1395 (size=9814) 2024-12-08T08:15:48,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,809 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120870157324ba194ab9b4e48bf7bad11f50_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120870157324ba194ab9b4e48bf7bad11f50_702b91089e160458511de9d558436b69 2024-12-08T08:15:48,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/57ac10c2d3714cdd9752d5c34980d7c0, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/57ac10c2d3714cdd9752d5c34980d7c0 is 175, key is test_row_0/A:col10/1733645747538/Put/seqid=0 2024-12-08T08:15:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742221_1397 (size=22461) 2024-12-08T08:15:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,815 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/57ac10c2d3714cdd9752d5c34980d7c0 2024-12-08T08:15:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/c0d576700c14452384576ef1c6b85e35 is 50, key is test_row_0/B:col10/1733645747538/Put/seqid=0 2024-12-08T08:15:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742222_1398 (size=9757) 2024-12-08T08:15:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,827 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/c0d576700c14452384576ef1c6b85e35 2024-12-08T08:15:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/cb1ed595b67e4e41848355ef3cd16c01 is 50, key is test_row_0/C:col10/1733645747538/Put/seqid=0 2024-12-08T08:15:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742223_1399 (size=9757) 2024-12-08T08:15:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,837 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/cb1ed595b67e4e41848355ef3cd16c01 2024-12-08T08:15:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/57ac10c2d3714cdd9752d5c34980d7c0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/57ac10c2d3714cdd9752d5c34980d7c0 2024-12-08T08:15:48,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,843 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/57ac10c2d3714cdd9752d5c34980d7c0, entries=100, sequenceid=153, filesize=21.9 K 2024-12-08T08:15:48,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/c0d576700c14452384576ef1c6b85e35 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/c0d576700c14452384576ef1c6b85e35 2024-12-08T08:15:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,847 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/c0d576700c14452384576ef1c6b85e35, entries=100, sequenceid=153, filesize=9.5 K 2024-12-08T08:15:48,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/cb1ed595b67e4e41848355ef3cd16c01 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/cb1ed595b67e4e41848355ef3cd16c01 2024-12-08T08:15:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,851 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/cb1ed595b67e4e41848355ef3cd16c01, entries=100, sequenceid=153, filesize=9.5 K 2024-12-08T08:15:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,852 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 702b91089e160458511de9d558436b69 in 87ms, sequenceid=153, compaction requested=true 2024-12-08T08:15:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-08T08:15:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-08T08:15:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-08T08:15:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4600 sec 2024-12-08T08:15:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.4650 sec 2024-12-08T08:15:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:49,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:15:49,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:49,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:49,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:49,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:49,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120809dc4e9effe840729f87ce2f4e5235aa_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645749014/Put/seqid=0 2024-12-08T08:15:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742224_1400 (size=17284) 2024-12-08T08:15:49,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733645749039, max timestamp=1733645749083Num uniques 859; 2024-12-08T08:15:49,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733645749039, max timestamp=1733645749083Num uniques 859; 2024-12-08T08:15:49,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733645749039, max timestamp=1733645749083Num uniques 859; 2024-12-08T08:15:49,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T08:15:49,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T08:15:49,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T08:15:49,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645809136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645809136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645809137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645809138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,186 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/a63a85ca96814e78a7dc14815c6290fa as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/a63a85ca96814e78a7dc14815c6290fa 2024-12-08T08:15:49,194 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/f340a390c5554798b29f132e44f71994 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f340a390c5554798b29f132e44f71994 2024-12-08T08:15:49,194 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 702b91089e160458511de9d558436b69/B of 702b91089e160458511de9d558436b69 into a63a85ca96814e78a7dc14815c6290fa(size=12.2 K), total size for store is 21.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:49,194 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:49,194 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/B, priority=12, startTime=1733645748759; duration=0sec 2024-12-08T08:15:49,194 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:49,194 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:B 2024-12-08T08:15:49,194 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-08T08:15:49,196 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58117 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-08T08:15:49,196 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/C is initiating minor compaction (all files) 2024-12-08T08:15:49,196 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/C in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:49,196 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/cd2f5b43c318461bac2ed4dd2de24fd8, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/70a79e46fe8a4a7fbcaee93128b049d7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/d5261a7fd4ee436fb2206bf65d966fe3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/414bf55d101748729dd6ef8e4b5a47ad, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/cb1ed595b67e4e41848355ef3cd16c01] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=56.8 K 2024-12-08T08:15:49,196 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2f5b43c318461bac2ed4dd2de24fd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733645741889 2024-12-08T08:15:49,196 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 70a79e46fe8a4a7fbcaee93128b049d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733645742547 2024-12-08T08:15:49,197 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting d5261a7fd4ee436fb2206bf65d966fe3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733645744715 2024-12-08T08:15:49,197 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 414bf55d101748729dd6ef8e4b5a47ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733645746829 2024-12-08T08:15:49,198 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting cb1ed595b67e4e41848355ef3cd16c01, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733645747528 2024-12-08T08:15:49,199 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 702b91089e160458511de9d558436b69/A of 702b91089e160458511de9d558436b69 into f340a390c5554798b29f132e44f71994(size=30.7 K), total size for store is 52.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:49,199 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:49,199 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/A, priority=12, startTime=1733645748759; duration=0sec 2024-12-08T08:15:49,199 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:49,199 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:A 2024-12-08T08:15:49,209 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:49,210 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/1d4737b6120144acb8d1d08d73477584 is 50, key is test_row_0/C:col10/1733645747538/Put/seqid=0 2024-12-08T08:15:49,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742225_1401 (size=12527) 2024-12-08T08:15:49,218 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/1d4737b6120144acb8d1d08d73477584 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/1d4737b6120144acb8d1d08d73477584 2024-12-08T08:15:49,222 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 702b91089e160458511de9d558436b69/C of 702b91089e160458511de9d558436b69 into 1d4737b6120144acb8d1d08d73477584(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:49,222 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:49,222 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/C, priority=11, startTime=1733645748759; duration=0sec 2024-12-08T08:15:49,223 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:49,223 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:C 2024-12-08T08:15:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645809250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645809250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645809250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645809261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,453 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:49,456 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120809dc4e9effe840729f87ce2f4e5235aa_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120809dc4e9effe840729f87ce2f4e5235aa_702b91089e160458511de9d558436b69 2024-12-08T08:15:49,457 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/0eea7b68ae694bb79463b009a0fbae4f, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:49,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/0eea7b68ae694bb79463b009a0fbae4f is 175, key is test_row_0/A:col10/1733645749014/Put/seqid=0 2024-12-08T08:15:49,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742226_1402 (size=48389) 2024-12-08T08:15:49,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645809460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645809460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645809471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645809471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T08:15:49,551 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:49,551 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-08T08:15:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T08:15:49,553 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:49,553 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:49,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T08:15:49,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645809701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,704 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 702b91089e160458511de9d558436b69 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:49,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:49,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:49,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:49,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:49,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:49,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:49,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645809770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645809781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645809781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645809781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:49,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:49,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:49,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:49,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:49,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:49,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:49,862 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/0eea7b68ae694bb79463b009a0fbae4f 2024-12-08T08:15:49,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/0edd1967783e45c6b87d4aa724f1d8a4 is 50, key is test_row_0/B:col10/1733645749014/Put/seqid=0 2024-12-08T08:15:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T08:15:49,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742227_1403 (size=12151) 2024-12-08T08:15:50,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:50,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:50,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:50,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:50,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T08:15:50,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/0edd1967783e45c6b87d4aa724f1d8a4 2024-12-08T08:15:50,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/c5ea226058724dd383cc56e15145aa3c is 50, key is test_row_0/C:col10/1733645749014/Put/seqid=0 2024-12-08T08:15:50,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742228_1404 (size=12151) 2024-12-08T08:15:50,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/c5ea226058724dd383cc56e15145aa3c 2024-12-08T08:15:50,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/0eea7b68ae694bb79463b009a0fbae4f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/0eea7b68ae694bb79463b009a0fbae4f 2024-12-08T08:15:50,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:50,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645810281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:50,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/0eea7b68ae694bb79463b009a0fbae4f, entries=250, sequenceid=163, filesize=47.3 K 2024-12-08T08:15:50,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/0edd1967783e45c6b87d4aa724f1d8a4 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/0edd1967783e45c6b87d4aa724f1d8a4 2024-12-08T08:15:50,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/0edd1967783e45c6b87d4aa724f1d8a4, entries=150, sequenceid=163, filesize=11.9 K 2024-12-08T08:15:50,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/c5ea226058724dd383cc56e15145aa3c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/c5ea226058724dd383cc56e15145aa3c 2024-12-08T08:15:50,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/c5ea226058724dd383cc56e15145aa3c, entries=150, sequenceid=163, filesize=11.9 K 2024-12-08T08:15:50,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 702b91089e160458511de9d558436b69 in 1284ms, sequenceid=163, compaction requested=true 2024-12-08T08:15:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:50,298 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:50,298 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:50,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:50,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-12-08T08:15:50,299 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:50,299 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/B is initiating minor compaction (all files) 2024-12-08T08:15:50,299 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/B in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,299 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/a63a85ca96814e78a7dc14815c6290fa, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/c0d576700c14452384576ef1c6b85e35, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/0edd1967783e45c6b87d4aa724f1d8a4] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=33.6 K 2024-12-08T08:15:50,299 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:50,299 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/A is initiating minor compaction (all files) 2024-12-08T08:15:50,299 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/A in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,299 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f340a390c5554798b29f132e44f71994, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/57ac10c2d3714cdd9752d5c34980d7c0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/0eea7b68ae694bb79463b009a0fbae4f] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=99.9 K 2024-12-08T08:15:50,299 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting a63a85ca96814e78a7dc14815c6290fa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733645746829 2024-12-08T08:15:50,299 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,300 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f340a390c5554798b29f132e44f71994, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/57ac10c2d3714cdd9752d5c34980d7c0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/0eea7b68ae694bb79463b009a0fbae4f] 2024-12-08T08:15:50,300 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d576700c14452384576ef1c6b85e35, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733645747528 2024-12-08T08:15:50,300 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting f340a390c5554798b29f132e44f71994, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733645746829 2024-12-08T08:15:50,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:50,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:50,300 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 0edd1967783e45c6b87d4aa724f1d8a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733645748970 2024-12-08T08:15:50,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:50,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:50,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:50,300 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57ac10c2d3714cdd9752d5c34980d7c0, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733645747528 2024-12-08T08:15:50,300 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eea7b68ae694bb79463b009a0fbae4f, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733645748970 2024-12-08T08:15:50,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:50,307 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:50,308 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:50,308 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/1ccd5a37604a48c781e8621168a4d6a0 is 50, key is test_row_0/B:col10/1733645749014/Put/seqid=0 2024-12-08T08:15:50,309 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208e2667b2e120a4e31b7f167304d217dc3_702b91089e160458511de9d558436b69 store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:50,311 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208e2667b2e120a4e31b7f167304d217dc3_702b91089e160458511de9d558436b69, store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:50,311 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e2667b2e120a4e31b7f167304d217dc3_702b91089e160458511de9d558436b69 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:50,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:50,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645810311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:50,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742229_1405 (size=12595) 2024-12-08T08:15:50,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:50,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:50,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742230_1406 (size=4469) 2024-12-08T08:15:50,316 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#A#compaction#343 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:50,317 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/0f5ce3f2ac694dccb20c74a66d75df01 is 175, key is test_row_0/A:col10/1733645749014/Put/seqid=0 2024-12-08T08:15:50,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dc9c0ac8380243aa9fb79d38626d0762_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645749137/Put/seqid=0 2024-12-08T08:15:50,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742231_1407 (size=31549) 2024-12-08T08:15:50,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742232_1408 (size=17284) 2024-12-08T08:15:50,321 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:50,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:50,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645810311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:50,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:50,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645810312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:50,324 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dc9c0ac8380243aa9fb79d38626d0762_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dc9c0ac8380243aa9fb79d38626d0762_702b91089e160458511de9d558436b69 2024-12-08T08:15:50,324 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/479bda047ab447d6b1296f879d9baf4b, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:50,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/479bda047ab447d6b1296f879d9baf4b is 175, key is test_row_0/A:col10/1733645749137/Put/seqid=0 2024-12-08T08:15:50,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742233_1409 (size=48389) 2024-12-08T08:15:50,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:50,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645810420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:50,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:50,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:50,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645810431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:50,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645810431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:50,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645810631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:50,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645810640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:50,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645810640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T08:15:50,717 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/1ccd5a37604a48c781e8621168a4d6a0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/1ccd5a37604a48c781e8621168a4d6a0 2024-12-08T08:15:50,722 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 702b91089e160458511de9d558436b69/B of 702b91089e160458511de9d558436b69 into 1ccd5a37604a48c781e8621168a4d6a0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:50,722 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:50,722 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/B, priority=13, startTime=1733645750298; duration=0sec 2024-12-08T08:15:50,722 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:50,722 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:B 2024-12-08T08:15:50,722 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-08T08:15:50,723 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-08T08:15:50,723 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-08T08:15:50,723 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. because compaction request was cancelled 2024-12-08T08:15:50,723 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:C 2024-12-08T08:15:50,724 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/0f5ce3f2ac694dccb20c74a66d75df01 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/0f5ce3f2ac694dccb20c74a66d75df01 2024-12-08T08:15:50,728 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 702b91089e160458511de9d558436b69/A of 702b91089e160458511de9d558436b69 into 0f5ce3f2ac694dccb20c74a66d75df01(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:50,728 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:50,728 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/A, priority=13, startTime=1733645750298; duration=0sec 2024-12-08T08:15:50,729 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/479bda047ab447d6b1296f879d9baf4b 2024-12-08T08:15:50,729 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:50,729 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:A 2024-12-08T08:15:50,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/3eed28c9af3749679933fbf9e458aafa is 50, key is test_row_0/B:col10/1733645749137/Put/seqid=0 2024-12-08T08:15:50,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742234_1410 (size=12151) 2024-12-08T08:15:50,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:50,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:50,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:50,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:50,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645810941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:50,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645810951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:50,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645810951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:51,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:51,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:51,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:51,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:51,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:51,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:51,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/3eed28c9af3749679933fbf9e458aafa 2024-12-08T08:15:51,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/b06c0191c6a94e68be973c56d4e7a170 is 50, key is test_row_0/C:col10/1733645749137/Put/seqid=0 2024-12-08T08:15:51,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742235_1411 (size=12151) 2024-12-08T08:15:51,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:51,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:51,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:51,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:51,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:51,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:51,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645811290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:51,297 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 702b91089e160458511de9d558436b69 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:51,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:51,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:51,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:51,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:51,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:51,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:51,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645811451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:51,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645811470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:51,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645811470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:51,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:51,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:51,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:51,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:51,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:51,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:51,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/b06c0191c6a94e68be973c56d4e7a170 2024-12-08T08:15:51,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/479bda047ab447d6b1296f879d9baf4b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/479bda047ab447d6b1296f879d9baf4b 2024-12-08T08:15:51,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/479bda047ab447d6b1296f879d9baf4b, entries=250, sequenceid=203, filesize=47.3 K 2024-12-08T08:15:51,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/3eed28c9af3749679933fbf9e458aafa as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3eed28c9af3749679933fbf9e458aafa 2024-12-08T08:15:51,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3eed28c9af3749679933fbf9e458aafa, entries=150, sequenceid=203, filesize=11.9 K 2024-12-08T08:15:51,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/b06c0191c6a94e68be973c56d4e7a170 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/b06c0191c6a94e68be973c56d4e7a170 2024-12-08T08:15:51,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/b06c0191c6a94e68be973c56d4e7a170, entries=150, sequenceid=203, filesize=11.9 K 2024-12-08T08:15:51,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 702b91089e160458511de9d558436b69 in 1271ms, sequenceid=203, compaction requested=true 2024-12-08T08:15:51,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:51,570 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-08T08:15:51,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:51,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:51,570 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-08T08:15:51,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:51,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:51,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:51,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:51,571 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-08T08:15:51,571 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-08T08:15:51,571 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-08T08:15:51,571 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-08T08:15:51,571 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. because compaction request was cancelled 2024-12-08T08:15:51,571 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. because compaction request was cancelled 2024-12-08T08:15:51,571 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:B 2024-12-08T08:15:51,571 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:A 2024-12-08T08:15:51,571 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:51,571 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:51,571 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/C is initiating minor compaction (all files) 2024-12-08T08:15:51,572 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/C in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:51,572 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/1d4737b6120144acb8d1d08d73477584, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/c5ea226058724dd383cc56e15145aa3c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/b06c0191c6a94e68be973c56d4e7a170] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=36.0 K 2024-12-08T08:15:51,572 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d4737b6120144acb8d1d08d73477584, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733645746840 2024-12-08T08:15:51,572 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5ea226058724dd383cc56e15145aa3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733645748970 2024-12-08T08:15:51,572 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting b06c0191c6a94e68be973c56d4e7a170, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733645749135 2024-12-08T08:15:51,579 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#C#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:51,579 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/1d93c3814930481c874e59d4f979f4af is 50, key is test_row_0/C:col10/1733645749137/Put/seqid=0 2024-12-08T08:15:51,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742236_1412 (size=12629) 2024-12-08T08:15:51,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T08:15:51,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:51,686 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T08:15:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080dcaa653cf414002b8b9234660bcd93a_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645750311/Put/seqid=0 2024-12-08T08:15:51,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742237_1413 (size=12304) 2024-12-08T08:15:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T08:15:51,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:51,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645751768, max timestamp=1733645751836Num uniques 859; 2024-12-08T08:15:51,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:15:51,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645751768, max timestamp=1733645751836Num uniques 859; 2024-12-08T08:15:51,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:15:51,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645751768, max timestamp=1733645751836Num uniques 859; 2024-12-08T08:15:51,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:15:51,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733645751949, max timestamp=1733645751967Num uniques 1018; 2024-12-08T08:15:51,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T08:15:51,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733645751949, max timestamp=1733645751967Num uniques 1018; 2024-12-08T08:15:51,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T08:15:51,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733645751949, max timestamp=1733645751967Num uniques 1018; 2024-12-08T08:15:51,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T08:15:51,992 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/1d93c3814930481c874e59d4f979f4af as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/1d93c3814930481c874e59d4f979f4af 2024-12-08T08:15:51,999 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 702b91089e160458511de9d558436b69/C of 702b91089e160458511de9d558436b69 into 1d93c3814930481c874e59d4f979f4af(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:52,000 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:52,000 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/C, priority=13, startTime=1733645751571; duration=0sec 2024-12-08T08:15:52,000 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:52,000 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:C 2024-12-08T08:15:52,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:52,099 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080dcaa653cf414002b8b9234660bcd93a_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080dcaa653cf414002b8b9234660bcd93a_702b91089e160458511de9d558436b69 2024-12-08T08:15:52,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/f6a64cced951400db1be59227b3eb204, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:52,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/f6a64cced951400db1be59227b3eb204 is 175, key is test_row_0/A:col10/1733645750311/Put/seqid=0 2024-12-08T08:15:52,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742238_1414 (size=31105) 2024-12-08T08:15:52,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733645752090, max timestamp=1733645752146Num uniques 1027; 2024-12-08T08:15:52,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733645752090, max timestamp=1733645752146Num uniques 1027; 2024-12-08T08:15:52,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-08T08:15:52,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-08T08:15:52,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733645752090, max timestamp=1733645752146Num uniques 1027; 2024-12-08T08:15:52,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-08T08:15:52,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733645752252, max timestamp=1733645752340Num uniques 1036; 2024-12-08T08:15:52,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-08T08:15:52,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733645752252, max timestamp=1733645752340Num uniques 1036; 2024-12-08T08:15:52,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-08T08:15:52,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733645752252, max timestamp=1733645752340Num uniques 1036; 2024-12-08T08:15:52,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-08T08:15:52,510 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/f6a64cced951400db1be59227b3eb204 2024-12-08T08:15:52,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733645752498, max timestamp=1733645752506Num uniques 1045; 2024-12-08T08:15:52,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-08T08:15:52,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733645752498, max timestamp=1733645752506Num uniques 1045; 2024-12-08T08:15:52,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-08T08:15:52,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733645752498, max timestamp=1733645752506Num uniques 1045; 2024-12-08T08:15:52,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-08T08:15:52,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/23a8a2ed65ff409cb126430490900fd9 is 50, key is test_row_0/B:col10/1733645750311/Put/seqid=0 2024-12-08T08:15:52,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742239_1415 (size=12151) 2024-12-08T08:15:52,544 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/23a8a2ed65ff409cb126430490900fd9 2024-12-08T08:15:52,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/9fabc11432a04fa6ba056e99b1019341 is 50, key is test_row_0/C:col10/1733645750311/Put/seqid=0 2024-12-08T08:15:52,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742240_1416 (size=12151) 2024-12-08T08:15:52,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733645752543, max timestamp=1733645752574Num uniques 1054; 2024-12-08T08:15:52,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656  suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-08T08:15:52,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733645752543, max timestamp=1733645752574Num uniques 1054; 2024-12-08T08:15:52,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733645752543, max timestamp=1733645752574Num uniques 1054; 2024-12-08T08:15:52,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656  suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-08T08:15:52,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656  suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-08T08:15:52,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645812631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:52,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645812632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:52,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:52,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645812647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:52,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645812649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:52,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645812760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:52,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645812761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645812771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645812771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:52,959 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/9fabc11432a04fa6ba056e99b1019341 2024-12-08T08:15:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/f6a64cced951400db1be59227b3eb204 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f6a64cced951400db1be59227b3eb204 2024-12-08T08:15:52,968 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f6a64cced951400db1be59227b3eb204, entries=150, sequenceid=213, filesize=30.4 K 2024-12-08T08:15:52,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/23a8a2ed65ff409cb126430490900fd9 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/23a8a2ed65ff409cb126430490900fd9 2024-12-08T08:15:52,973 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/23a8a2ed65ff409cb126430490900fd9, entries=150, sequenceid=213, filesize=11.9 K 2024-12-08T08:15:52,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/9fabc11432a04fa6ba056e99b1019341 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/9fabc11432a04fa6ba056e99b1019341 2024-12-08T08:15:52,982 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/9fabc11432a04fa6ba056e99b1019341, entries=150, sequenceid=213, filesize=11.9 K 2024-12-08T08:15:52,983 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 702b91089e160458511de9d558436b69 in 1297ms, sequenceid=213, compaction requested=true 2024-12-08T08:15:52,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:52,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:52,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-08T08:15:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-08T08:15:52,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-08T08:15:52,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4320 sec 2024-12-08T08:15:52,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.4380 sec 2024-12-08T08:15:52,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=154.31 KB heapSize=436.55 KB 2024-12-08T08:15:52,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:52,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T08:15:52,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:52,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T08:15:52,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:52,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T08:15:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:53,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645812995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645812998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645813001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f658424bd6164826aa62b0bf78f87cce_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645752644/Put/seqid=0 2024-12-08T08:15:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645813012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742241_1417 (size=14994) 2024-12-08T08:15:53,054 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:53,065 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f658424bd6164826aa62b0bf78f87cce_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f658424bd6164826aa62b0bf78f87cce_702b91089e160458511de9d558436b69 2024-12-08T08:15:53,066 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/8f4ca2d11b3e41bbb1de4fa3d844c109, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:53,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/8f4ca2d11b3e41bbb1de4fa3d844c109 is 175, key is test_row_0/A:col10/1733645752644/Put/seqid=0 2024-12-08T08:15:53,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742242_1418 (size=39949) 2024-12-08T08:15:53,075 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/8f4ca2d11b3e41bbb1de4fa3d844c109 2024-12-08T08:15:53,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/976150b6156c4924af2922120e31d3f4 is 50, key is test_row_0/B:col10/1733645752644/Put/seqid=0 2024-12-08T08:15:53,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742243_1419 (size=12301) 2024-12-08T08:15:53,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/976150b6156c4924af2922120e31d3f4 2024-12-08T08:15:53,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/a937d4271e534a499210cd853f9e9f7a is 50, key is test_row_0/C:col10/1733645752644/Put/seqid=0 2024-12-08T08:15:53,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645813122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645813122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645813122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645813131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742244_1420 (size=12301) 2024-12-08T08:15:53,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645813320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,329 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 702b91089e160458511de9d558436b69 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645813341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645813341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645813341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645813350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/a937d4271e534a499210cd853f9e9f7a 2024-12-08T08:15:53,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/8f4ca2d11b3e41bbb1de4fa3d844c109 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8f4ca2d11b3e41bbb1de4fa3d844c109 2024-12-08T08:15:53,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8f4ca2d11b3e41bbb1de4fa3d844c109, entries=200, sequenceid=338, filesize=39.0 K 2024-12-08T08:15:53,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/976150b6156c4924af2922120e31d3f4 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/976150b6156c4924af2922120e31d3f4 2024-12-08T08:15:53,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/976150b6156c4924af2922120e31d3f4, entries=150, sequenceid=338, filesize=12.0 K 2024-12-08T08:15:53,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/a937d4271e534a499210cd853f9e9f7a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/a937d4271e534a499210cd853f9e9f7a 2024-12-08T08:15:53,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/a937d4271e534a499210cd853f9e9f7a, entries=150, sequenceid=338, filesize=12.0 K 2024-12-08T08:15:53,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.50 KB/446976, currentSize=26.84 KB/27480 for 702b91089e160458511de9d558436b69 in 593ms, sequenceid=338, compaction requested=true 2024-12-08T08:15:53,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:53,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:53,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:53,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:53,585 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:15:53,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:53,585 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:15:53,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:53,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:53,586 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:15:53,586 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150992 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:15:53,586 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/B is initiating minor compaction (all files) 2024-12-08T08:15:53,586 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/A is initiating minor compaction (all files) 2024-12-08T08:15:53,586 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/B in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:53,586 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/A in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:53,586 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/0f5ce3f2ac694dccb20c74a66d75df01, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/479bda047ab447d6b1296f879d9baf4b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f6a64cced951400db1be59227b3eb204, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8f4ca2d11b3e41bbb1de4fa3d844c109] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=147.5 K 2024-12-08T08:15:53,586 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/1ccd5a37604a48c781e8621168a4d6a0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3eed28c9af3749679933fbf9e458aafa, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/23a8a2ed65ff409cb126430490900fd9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/976150b6156c4924af2922120e31d3f4] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=48.0 K 2024-12-08T08:15:53,586 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:53,587 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/0f5ce3f2ac694dccb20c74a66d75df01, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/479bda047ab447d6b1296f879d9baf4b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f6a64cced951400db1be59227b3eb204, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8f4ca2d11b3e41bbb1de4fa3d844c109] 2024-12-08T08:15:53,588 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ccd5a37604a48c781e8621168a4d6a0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733645748970 2024-12-08T08:15:53,588 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f5ce3f2ac694dccb20c74a66d75df01, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733645748970 2024-12-08T08:15:53,588 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 479bda047ab447d6b1296f879d9baf4b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733645749133 2024-12-08T08:15:53,588 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eed28c9af3749679933fbf9e458aafa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733645749135 2024-12-08T08:15:53,590 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6a64cced951400db1be59227b3eb204, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733645750309 2024-12-08T08:15:53,592 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 23a8a2ed65ff409cb126430490900fd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733645750309 2024-12-08T08:15:53,592 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f4ca2d11b3e41bbb1de4fa3d844c109, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733645752630 2024-12-08T08:15:53,592 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 976150b6156c4924af2922120e31d3f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733645752630 2024-12-08T08:15:53,601 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:53,603 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#B#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:53,603 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/ff30a3c9c81c4e00a36d6ab8a2410e69 is 50, key is test_row_0/B:col10/1733645752644/Put/seqid=0 2024-12-08T08:15:53,613 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120895c9ae97234a406cb906ef2d3f8fc86c_702b91089e160458511de9d558436b69 store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:53,615 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120895c9ae97234a406cb906ef2d3f8fc86c_702b91089e160458511de9d558436b69, store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:53,615 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120895c9ae97234a406cb906ef2d3f8fc86c_702b91089e160458511de9d558436b69 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:53,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742245_1421 (size=12881) 2024-12-08T08:15:53,640 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/ff30a3c9c81c4e00a36d6ab8a2410e69 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/ff30a3c9c81c4e00a36d6ab8a2410e69 2024-12-08T08:15:53,645 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 702b91089e160458511de9d558436b69/B of 702b91089e160458511de9d558436b69 into ff30a3c9c81c4e00a36d6ab8a2410e69(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:53,645 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:53,645 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/B, priority=12, startTime=1733645753585; duration=0sec 2024-12-08T08:15:53,645 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:53,645 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:B 2024-12-08T08:15:53,645 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:15:53,646 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:15:53,647 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/C is initiating minor compaction (all files) 2024-12-08T08:15:53,647 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/C in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:53,647 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/1d93c3814930481c874e59d4f979f4af, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/9fabc11432a04fa6ba056e99b1019341, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/a937d4271e534a499210cd853f9e9f7a] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=36.2 K 2024-12-08T08:15:53,647 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d93c3814930481c874e59d4f979f4af, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733645749135 2024-12-08T08:15:53,648 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fabc11432a04fa6ba056e99b1019341, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733645750309 2024-12-08T08:15:53,649 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting a937d4271e534a499210cd853f9e9f7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733645752630 2024-12-08T08:15:53,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:15:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:53,670 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#C#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:53,670 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/7c5663cae18c4f888a8f256aa0a76cd3 is 50, key is test_row_0/C:col10/1733645752644/Put/seqid=0 2024-12-08T08:15:53,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742246_1422 (size=4469) 2024-12-08T08:15:53,686 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#A#compaction#354 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:53,687 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/a69d1a98e75f46758f4a595262d9a923 is 175, key is test_row_0/A:col10/1733645752644/Put/seqid=0 2024-12-08T08:15:53,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742247_1423 (size=12881) 2024-12-08T08:15:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T08:15:53,731 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:53,731 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:15:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-08T08:15:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T08:15:53,733 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:15:53,734 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:15:53,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:15:53,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645753715, max timestamp=1733645753733Num uniques 859; 2024-12-08T08:15:53,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:15:53,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645753715, max timestamp=1733645753733Num uniques 859; 2024-12-08T08:15:53,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:15:53,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645753715, max timestamp=1733645753733Num uniques 859; 2024-12-08T08:15:53,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:15:53,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742248_1424 (size=31835) 2024-12-08T08:15:53,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a0477aba97c5445ebfdf528b345adfdb_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645753656/Put/seqid=0 2024-12-08T08:15:53,743 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/a69d1a98e75f46758f4a595262d9a923 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/a69d1a98e75f46758f4a595262d9a923 2024-12-08T08:15:53,750 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 702b91089e160458511de9d558436b69/A of 702b91089e160458511de9d558436b69 into a69d1a98e75f46758f4a595262d9a923(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:53,750 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:53,750 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/A, priority=12, startTime=1733645753585; duration=0sec 2024-12-08T08:15:53,750 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:53,750 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:A 2024-12-08T08:15:53,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742249_1425 (size=14994) 2024-12-08T08:15:53,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645813777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645813778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645813780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645813783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T08:15:53,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T08:15:53,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:53,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:53,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:53,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:53,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:53,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645813891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645813900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645813900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645813901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:54,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T08:15:54,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:54,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T08:15:54,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645814111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:54,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645814111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645814111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:54,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645814111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:54,125 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/7c5663cae18c4f888a8f256aa0a76cd3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/7c5663cae18c4f888a8f256aa0a76cd3 2024-12-08T08:15:54,130 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 702b91089e160458511de9d558436b69/C of 702b91089e160458511de9d558436b69 into 7c5663cae18c4f888a8f256aa0a76cd3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:54,130 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:54,130 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/C, priority=13, startTime=1733645753585; duration=0sec 2024-12-08T08:15:54,130 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:54,130 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:C 2024-12-08T08:15:54,154 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:54,157 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a0477aba97c5445ebfdf528b345adfdb_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a0477aba97c5445ebfdf528b345adfdb_702b91089e160458511de9d558436b69 2024-12-08T08:15:54,158 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/994d7fb1f31b429da10a9bae9cb1cb53, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:54,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/994d7fb1f31b429da10a9bae9cb1cb53 is 175, key is test_row_0/A:col10/1733645753656/Put/seqid=0 2024-12-08T08:15:54,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742250_1426 (size=39949) 2024-12-08T08:15:54,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T08:15:54,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:54,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T08:15:54,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:54,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T08:15:54,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645814430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645814431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:54,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645814431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645814431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:54,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T08:15:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,563 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/994d7fb1f31b429da10a9bae9cb1cb53 2024-12-08T08:15:54,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/d422a238e6bd4f8ba3159313f21a0739 is 50, key is test_row_0/B:col10/1733645753656/Put/seqid=0 2024-12-08T08:15:54,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742251_1427 (size=12301) 2024-12-08T08:15:54,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T08:15:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T08:15:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T08:15:54,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645814941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:54,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645814941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:54,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645814941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645814941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:54,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T08:15:54,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:54,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:15:54,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/d422a238e6bd4f8ba3159313f21a0739 2024-12-08T08:15:54,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/464d9175f4c94613a2ccf8560f3b2fce is 50, key is test_row_0/C:col10/1733645753656/Put/seqid=0 2024-12-08T08:15:54,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742252_1428 (size=12301) 2024-12-08T08:15:54,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/464d9175f4c94613a2ccf8560f3b2fce 2024-12-08T08:15:54,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/994d7fb1f31b429da10a9bae9cb1cb53 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/994d7fb1f31b429da10a9bae9cb1cb53 2024-12-08T08:15:55,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/994d7fb1f31b429da10a9bae9cb1cb53, entries=200, sequenceid=349, filesize=39.0 K 2024-12-08T08:15:55,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/d422a238e6bd4f8ba3159313f21a0739 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/d422a238e6bd4f8ba3159313f21a0739 2024-12-08T08:15:55,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/d422a238e6bd4f8ba3159313f21a0739, entries=150, sequenceid=349, filesize=12.0 K 2024-12-08T08:15:55,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/464d9175f4c94613a2ccf8560f3b2fce as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/464d9175f4c94613a2ccf8560f3b2fce 2024-12-08T08:15:55,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/464d9175f4c94613a2ccf8560f3b2fce, entries=150, sequenceid=349, filesize=12.0 K 2024-12-08T08:15:55,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 702b91089e160458511de9d558436b69 in 1347ms, sequenceid=349, compaction requested=false 2024-12-08T08:15:55,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:55,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T08:15:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:55,113 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-08T08:15:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:55,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208334ae6a58f9f4ddbbf92d15e344e5330_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645753766/Put/seqid=0 2024-12-08T08:15:55,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742253_1429 (size=12454) 2024-12-08T08:15:55,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:55,534 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208334ae6a58f9f4ddbbf92d15e344e5330_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208334ae6a58f9f4ddbbf92d15e344e5330_702b91089e160458511de9d558436b69 2024-12-08T08:15:55,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/6fd22c9c16a4477298b7e750c997d505, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:55,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/6fd22c9c16a4477298b7e750c997d505 is 175, key is test_row_0/A:col10/1733645753766/Put/seqid=0 2024-12-08T08:15:55,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742254_1430 (size=31255) 2024-12-08T08:15:55,562 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/6fd22c9c16a4477298b7e750c997d505 2024-12-08T08:15:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/62baaa59027f4f2badb1e3bb2676e400 is 50, key is test_row_0/B:col10/1733645753766/Put/seqid=0 2024-12-08T08:15:55,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742255_1431 (size=12301) 2024-12-08T08:15:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T08:15:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:55,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. as already flushing 2024-12-08T08:15:55,977 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/62baaa59027f4f2badb1e3bb2676e400 2024-12-08T08:15:55,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/721b7028fc61446ea0eb2fe0713e42c3 is 50, key is test_row_0/C:col10/1733645753766/Put/seqid=0 2024-12-08T08:15:55,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645815977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:55,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645815978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:55,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645815979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:55,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:55,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645815981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:55,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742256_1432 (size=12301) 2024-12-08T08:15:56,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645816091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645816101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645816101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645816101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645816300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645816320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645816321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645816321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,394 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/721b7028fc61446ea0eb2fe0713e42c3 2024-12-08T08:15:56,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/6fd22c9c16a4477298b7e750c997d505 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6fd22c9c16a4477298b7e750c997d505 2024-12-08T08:15:56,401 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6fd22c9c16a4477298b7e750c997d505, entries=150, sequenceid=387, filesize=30.5 K 2024-12-08T08:15:56,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/62baaa59027f4f2badb1e3bb2676e400 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/62baaa59027f4f2badb1e3bb2676e400 2024-12-08T08:15:56,405 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/62baaa59027f4f2badb1e3bb2676e400, entries=150, sequenceid=387, filesize=12.0 K 2024-12-08T08:15:56,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/721b7028fc61446ea0eb2fe0713e42c3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/721b7028fc61446ea0eb2fe0713e42c3 2024-12-08T08:15:56,409 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/721b7028fc61446ea0eb2fe0713e42c3, entries=150, sequenceid=387, filesize=12.0 K 2024-12-08T08:15:56,410 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 702b91089e160458511de9d558436b69 in 1297ms, sequenceid=387, compaction requested=true 2024-12-08T08:15:56,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:56,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:56,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-08T08:15:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-08T08:15:56,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-08T08:15:56,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6770 sec 2024-12-08T08:15:56,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.6810 sec 2024-12-08T08:15:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:56,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T08:15:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:15:56,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208277ecb747eb949a4bc39134980c35824_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645755977/Put/seqid=0 2024-12-08T08:15:56,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742257_1433 (size=14994) 2024-12-08T08:15:56,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733645756667, max timestamp=1733645756693Num uniques 859; 2024-12-08T08:15:56,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:15:56,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733645756667, max timestamp=1733645756693Num uniques 859; 2024-12-08T08:15:56,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:15:56,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733645756667, max timestamp=1733645756693Num uniques 859; 2024-12-08T08:15:56,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:15:56,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645816729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645816742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645816742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645816743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645816850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645816850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645816860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645816860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:56,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:57,046 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:57,049 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208277ecb747eb949a4bc39134980c35824_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208277ecb747eb949a4bc39134980c35824_702b91089e160458511de9d558436b69 2024-12-08T08:15:57,050 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/d3f0626ad8f84605a9d3fe444d5aea9b, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:57,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/d3f0626ad8f84605a9d3fe444d5aea9b is 175, key is test_row_0/A:col10/1733645755977/Put/seqid=0 2024-12-08T08:15:57,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742258_1434 (size=39949) 2024-12-08T08:15:57,055 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/d3f0626ad8f84605a9d3fe444d5aea9b 2024-12-08T08:15:57,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/a45fef5dedb4457bab74054fed1daa10 is 50, key is test_row_0/B:col10/1733645755977/Put/seqid=0 2024-12-08T08:15:57,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:57,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645817070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645817070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:57,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645817081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:57,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:57,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742259_1435 (size=12301) 2024-12-08T08:15:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645817081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:57,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/a45fef5dedb4457bab74054fed1daa10 2024-12-08T08:15:57,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/9e0027e1d9464ed3b3b8062f02bb210c is 50, key is test_row_0/C:col10/1733645755977/Put/seqid=0 2024-12-08T08:15:57,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742260_1436 (size=12301) 2024-12-08T08:15:57,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36694 deadline: 1733645817361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:57,371 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 702b91089e160458511de9d558436b69 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:15:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:57,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36648 deadline: 1733645817390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:57,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:57,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36714 deadline: 1733645817390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:57,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:57,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:57,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36688 deadline: 1733645817390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:57,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:15:57,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36704 deadline: 1733645817390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 2024-12-08T08:15:57,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:15:57,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:15:57,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/9e0027e1d9464ed3b3b8062f02bb210c 2024-12-08T08:15:57,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/d3f0626ad8f84605a9d3fe444d5aea9b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/d3f0626ad8f84605a9d3fe444d5aea9b 2024-12-08T08:15:57,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/d3f0626ad8f84605a9d3fe444d5aea9b, entries=200, sequenceid=398, filesize=39.0 K 2024-12-08T08:15:57,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/a45fef5dedb4457bab74054fed1daa10 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/a45fef5dedb4457bab74054fed1daa10 2024-12-08T08:15:57,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/a45fef5dedb4457bab74054fed1daa10, entries=150, sequenceid=398, filesize=12.0 K 2024-12-08T08:15:57,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/9e0027e1d9464ed3b3b8062f02bb210c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/9e0027e1d9464ed3b3b8062f02bb210c 2024-12-08T08:15:57,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/9e0027e1d9464ed3b3b8062f02bb210c, entries=150, sequenceid=398, filesize=12.0 K 2024-12-08T08:15:57,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 702b91089e160458511de9d558436b69 in 906ms, sequenceid=398, compaction requested=true 2024-12-08T08:15:57,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:57,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:15:57,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:57,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:15:57,526 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:15:57,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:57,526 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:15:57,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 702b91089e160458511de9d558436b69:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:15:57,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:57,527 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:15:57,527 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:15:57,527 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/B is initiating minor compaction (all files) 2024-12-08T08:15:57,527 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/A is initiating minor compaction (all files) 2024-12-08T08:15:57,527 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/B in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:57,527 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/A in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:57,527 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/ff30a3c9c81c4e00a36d6ab8a2410e69, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/d422a238e6bd4f8ba3159313f21a0739, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/62baaa59027f4f2badb1e3bb2676e400, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/a45fef5dedb4457bab74054fed1daa10] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=48.6 K 2024-12-08T08:15:57,527 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/a69d1a98e75f46758f4a595262d9a923, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/994d7fb1f31b429da10a9bae9cb1cb53, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6fd22c9c16a4477298b7e750c997d505, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/d3f0626ad8f84605a9d3fe444d5aea9b] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=139.6 K 2024-12-08T08:15:57,527 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:57,527 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/a69d1a98e75f46758f4a595262d9a923, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/994d7fb1f31b429da10a9bae9cb1cb53, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6fd22c9c16a4477298b7e750c997d505, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/d3f0626ad8f84605a9d3fe444d5aea9b] 2024-12-08T08:15:57,527 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting ff30a3c9c81c4e00a36d6ab8a2410e69, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733645752630 2024-12-08T08:15:57,527 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting a69d1a98e75f46758f4a595262d9a923, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733645752630 2024-12-08T08:15:57,527 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting d422a238e6bd4f8ba3159313f21a0739, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733645753655 2024-12-08T08:15:57,527 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 62baaa59027f4f2badb1e3bb2676e400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733645753766 2024-12-08T08:15:57,527 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 994d7fb1f31b429da10a9bae9cb1cb53, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733645753008 2024-12-08T08:15:57,528 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting a45fef5dedb4457bab74054fed1daa10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1733645755977 2024-12-08T08:15:57,528 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fd22c9c16a4477298b7e750c997d505, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733645753766 2024-12-08T08:15:57,528 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3f0626ad8f84605a9d3fe444d5aea9b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1733645755977 2024-12-08T08:15:57,534 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:57,535 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120858887b7431c44e1cae5a0453e4a01c16_702b91089e160458511de9d558436b69 store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:57,535 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:57,536 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/2cdc8c52eec74cb787414f8d4184a35e is 50, key is test_row_0/B:col10/1733645755977/Put/seqid=0 2024-12-08T08:15:57,536 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120858887b7431c44e1cae5a0453e4a01c16_702b91089e160458511de9d558436b69, store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:57,536 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120858887b7431c44e1cae5a0453e4a01c16_702b91089e160458511de9d558436b69 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:57,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742262_1438 (size=4469) 2024-12-08T08:15:57,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742261_1437 (size=13017) 2024-12-08T08:15:57,545 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#A#compaction#367 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:57,545 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/66398e8fcfca4cd7b787734840281808 is 175, key is test_row_0/A:col10/1733645755977/Put/seqid=0 2024-12-08T08:15:57,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742263_1439 (size=31971) 2024-12-08T08:15:57,552 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/66398e8fcfca4cd7b787734840281808 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/66398e8fcfca4cd7b787734840281808 2024-12-08T08:15:57,559 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 702b91089e160458511de9d558436b69/A of 702b91089e160458511de9d558436b69 into 66398e8fcfca4cd7b787734840281808(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:57,559 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:57,559 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/A, priority=12, startTime=1733645757526; duration=0sec 2024-12-08T08:15:57,559 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:15:57,559 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:A 2024-12-08T08:15:57,559 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:15:57,560 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:15:57,560 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 702b91089e160458511de9d558436b69/C is initiating minor compaction (all files) 2024-12-08T08:15:57,560 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 702b91089e160458511de9d558436b69/C in TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:15:57,560 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/7c5663cae18c4f888a8f256aa0a76cd3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/464d9175f4c94613a2ccf8560f3b2fce, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/721b7028fc61446ea0eb2fe0713e42c3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/9e0027e1d9464ed3b3b8062f02bb210c] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp, totalSize=48.6 K 2024-12-08T08:15:57,560 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c5663cae18c4f888a8f256aa0a76cd3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733645752630 2024-12-08T08:15:57,560 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 464d9175f4c94613a2ccf8560f3b2fce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733645753655 2024-12-08T08:15:57,561 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 721b7028fc61446ea0eb2fe0713e42c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733645753766 2024-12-08T08:15:57,561 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e0027e1d9464ed3b3b8062f02bb210c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1733645755977 2024-12-08T08:15:57,568 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 702b91089e160458511de9d558436b69#C#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:15:57,569 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/db6d86db12b446f0bb8d38e236bf32a3 is 50, key is test_row_0/C:col10/1733645755977/Put/seqid=0 2024-12-08T08:15:57,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742264_1440 (size=13017) 2024-12-08T08:15:57,788 INFO [Thread-1612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1612. 2024-12-08T08:15:57,788 DEBUG [Thread-1612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:57,788 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,788 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:57,788 INFO [Thread-1610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1610. 2024-12-08T08:15:57,788 DEBUG [Thread-1610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:57,788 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,788 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:57,789 INFO [Thread-1618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1618. 2024-12-08T08:15:57,789 DEBUG [Thread-1618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:57,789 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,789 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:57,794 INFO [Thread-1616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1616. 2024-12-08T08:15:57,794 DEBUG [Thread-1616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:57,794 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,794 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:57,795 INFO [Thread-1614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1614. 2024-12-08T08:15:57,795 DEBUG [Thread-1614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:57,795 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,795 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T08:15:57,902 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:15:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 702b91089e160458511de9d558436b69 2024-12-08T08:15:57,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-08T08:15:57,915 INFO [Thread-1607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1607. 2024-12-08T08:15:57,915 DEBUG [Thread-1607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:57,915 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,915 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:57,915 INFO [Thread-1601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1601. 2024-12-08T08:15:57,916 DEBUG [Thread-1601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:57,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:15:57,916 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,916 INFO [Thread-1599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1599. 2024-12-08T08:15:57,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:57,916 DEBUG [Thread-1599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:57,916 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:15:57,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:57,916 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:57,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:15:57,916 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:15:57,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:57,918 INFO [Thread-1603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1603. 2024-12-08T08:15:57,918 DEBUG [Thread-1603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:15:57,919 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,919 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:15:57,919 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:15:57,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120841f5962a7ead489899f6f3a9555fa3fa_702b91089e160458511de9d558436b69 is 50, key is test_row_0/A:col10/1733645757912/Put/seqid=0 2024-12-08T08:15:57,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742265_1441 (size=12454) 2024-12-08T08:15:57,946 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/2cdc8c52eec74cb787414f8d4184a35e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/2cdc8c52eec74cb787414f8d4184a35e 2024-12-08T08:15:57,952 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 702b91089e160458511de9d558436b69/B of 702b91089e160458511de9d558436b69 into 2cdc8c52eec74cb787414f8d4184a35e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:57,952 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:57,952 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/B, priority=12, startTime=1733645757526; duration=0sec 2024-12-08T08:15:57,952 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:57,952 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:B 2024-12-08T08:15:57,978 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/db6d86db12b446f0bb8d38e236bf32a3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/db6d86db12b446f0bb8d38e236bf32a3 2024-12-08T08:15:57,982 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 702b91089e160458511de9d558436b69/C of 702b91089e160458511de9d558436b69 into db6d86db12b446f0bb8d38e236bf32a3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:15:57,982 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:15:57,982 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., storeName=702b91089e160458511de9d558436b69/C, priority=12, startTime=1733645757526; duration=0sec 2024-12-08T08:15:57,982 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:15:57,982 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 702b91089e160458511de9d558436b69:C 2024-12-08T08:15:58,326 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:15:58,329 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120841f5962a7ead489899f6f3a9555fa3fa_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120841f5962a7ead489899f6f3a9555fa3fa_702b91089e160458511de9d558436b69 2024-12-08T08:15:58,329 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/9153811e543b4d1ab8e775e871aae4f9, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:15:58,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/9153811e543b4d1ab8e775e871aae4f9 is 175, key is test_row_0/A:col10/1733645757912/Put/seqid=0 2024-12-08T08:15:58,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742266_1442 (size=31255) 2024-12-08T08:15:58,737 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=436, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/9153811e543b4d1ab8e775e871aae4f9 2024-12-08T08:15:58,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/393cf5b5d97d49ec91465b9e3f42096f is 50, key is test_row_0/B:col10/1733645757912/Put/seqid=0 2024-12-08T08:15:58,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742267_1443 (size=12301) 2024-12-08T08:15:59,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/393cf5b5d97d49ec91465b9e3f42096f 2024-12-08T08:15:59,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/899b78fa180e4a7795d0cabd6eed07cf is 50, key is test_row_0/C:col10/1733645757912/Put/seqid=0 2024-12-08T08:15:59,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742268_1444 (size=12301) 2024-12-08T08:15:59,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/899b78fa180e4a7795d0cabd6eed07cf 2024-12-08T08:15:59,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/9153811e543b4d1ab8e775e871aae4f9 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/9153811e543b4d1ab8e775e871aae4f9 2024-12-08T08:15:59,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/9153811e543b4d1ab8e775e871aae4f9, entries=150, sequenceid=436, filesize=30.5 K 2024-12-08T08:15:59,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/393cf5b5d97d49ec91465b9e3f42096f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/393cf5b5d97d49ec91465b9e3f42096f 2024-12-08T08:15:59,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/393cf5b5d97d49ec91465b9e3f42096f, entries=150, sequenceid=436, filesize=12.0 K 2024-12-08T08:15:59,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/899b78fa180e4a7795d0cabd6eed07cf as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/899b78fa180e4a7795d0cabd6eed07cf 2024-12-08T08:15:59,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/899b78fa180e4a7795d0cabd6eed07cf, entries=150, sequenceid=436, filesize=12.0 K 2024-12-08T08:15:59,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=6.71 KB/6870 for 702b91089e160458511de9d558436b69 in 1670ms, sequenceid=436, compaction requested=false 2024-12-08T08:15:59,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 702b91089e160458511de9d558436b69: 2024-12-08T08:16:00,407 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T08:16:04,888 WARN [regionserver/422e41b75614:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-12-08T08:16:04,895 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/c2fd7aa63bb8410b977cd5f42114992b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/69ec92ee857c475891bb278fa399f12f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/a876237665464f3089687f8ea93968fa, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/781a933695934cd98ce2a6fc443ce009, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8efa10d408c842ba8e19155f5574906b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/dc212aefab4a443eacd0f2e12c96d0a0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f380eb2dc1d8403587fc4c7bd4bd558b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/5799b13559f740b0bf2cbe4620c9cd51, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6a41d1e0335144e9a2a3dbf66c94478c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/ee8a75f2e84a4980b891c3b59e713be0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f340a390c5554798b29f132e44f71994, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/57ac10c2d3714cdd9752d5c34980d7c0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/0eea7b68ae694bb79463b009a0fbae4f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/0f5ce3f2ac694dccb20c74a66d75df01, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/479bda047ab447d6b1296f879d9baf4b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f6a64cced951400db1be59227b3eb204, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8f4ca2d11b3e41bbb1de4fa3d844c109, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/a69d1a98e75f46758f4a595262d9a923, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/994d7fb1f31b429da10a9bae9cb1cb53, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6fd22c9c16a4477298b7e750c997d505, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/d3f0626ad8f84605a9d3fe444d5aea9b] to archive 2024-12-08T08:16:04,896 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:16:04,897 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/c2fd7aa63bb8410b977cd5f42114992b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/c2fd7aa63bb8410b977cd5f42114992b 2024-12-08T08:16:04,899 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/69ec92ee857c475891bb278fa399f12f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/69ec92ee857c475891bb278fa399f12f 2024-12-08T08:16:04,899 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/a876237665464f3089687f8ea93968fa to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/a876237665464f3089687f8ea93968fa 2024-12-08T08:16:04,900 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/781a933695934cd98ce2a6fc443ce009 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/781a933695934cd98ce2a6fc443ce009 2024-12-08T08:16:04,902 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8efa10d408c842ba8e19155f5574906b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8efa10d408c842ba8e19155f5574906b 2024-12-08T08:16:04,903 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/dc212aefab4a443eacd0f2e12c96d0a0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/dc212aefab4a443eacd0f2e12c96d0a0 2024-12-08T08:16:04,904 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f380eb2dc1d8403587fc4c7bd4bd558b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f380eb2dc1d8403587fc4c7bd4bd558b 2024-12-08T08:16:04,905 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/5799b13559f740b0bf2cbe4620c9cd51 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/5799b13559f740b0bf2cbe4620c9cd51 2024-12-08T08:16:04,906 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6a41d1e0335144e9a2a3dbf66c94478c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6a41d1e0335144e9a2a3dbf66c94478c 2024-12-08T08:16:04,907 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/ee8a75f2e84a4980b891c3b59e713be0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/ee8a75f2e84a4980b891c3b59e713be0 2024-12-08T08:16:04,908 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f340a390c5554798b29f132e44f71994 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f340a390c5554798b29f132e44f71994 2024-12-08T08:16:04,908 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/57ac10c2d3714cdd9752d5c34980d7c0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/57ac10c2d3714cdd9752d5c34980d7c0 2024-12-08T08:16:04,909 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/0eea7b68ae694bb79463b009a0fbae4f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/0eea7b68ae694bb79463b009a0fbae4f 2024-12-08T08:16:04,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/0f5ce3f2ac694dccb20c74a66d75df01 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/0f5ce3f2ac694dccb20c74a66d75df01 2024-12-08T08:16:04,911 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/479bda047ab447d6b1296f879d9baf4b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/479bda047ab447d6b1296f879d9baf4b 2024-12-08T08:16:04,912 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f6a64cced951400db1be59227b3eb204 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/f6a64cced951400db1be59227b3eb204 2024-12-08T08:16:04,912 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8f4ca2d11b3e41bbb1de4fa3d844c109 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/8f4ca2d11b3e41bbb1de4fa3d844c109 2024-12-08T08:16:04,913 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/a69d1a98e75f46758f4a595262d9a923 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/a69d1a98e75f46758f4a595262d9a923 2024-12-08T08:16:04,914 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/994d7fb1f31b429da10a9bae9cb1cb53 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/994d7fb1f31b429da10a9bae9cb1cb53 2024-12-08T08:16:04,915 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6fd22c9c16a4477298b7e750c997d505 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/6fd22c9c16a4477298b7e750c997d505 2024-12-08T08:16:04,916 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/d3f0626ad8f84605a9d3fe444d5aea9b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/d3f0626ad8f84605a9d3fe444d5aea9b 2024-12-08T08:16:04,918 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3ac37d2cc3fa4c40ae79a9bbb1833e8a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/25554c7f61d84f52ba3e521f42d0c064, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/534cd88268cc44caa4764ff749231827, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/ae4e55bdd8bf44bd80158efdd3f792fa, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/88e12fbb11664b68878266b31d73571d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/2c47114679c042d8916a27e7eaa82578, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/aa52c08b730a4365a43c5e70229dc0a2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/0e06da802d814bc5aa2a62a38da89d82, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3cee63bfcd9845c78a7d10c60c23e275, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/a63a85ca96814e78a7dc14815c6290fa, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/e017d72fbaa849fbb7d813a21c5df195, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/c0d576700c14452384576ef1c6b85e35, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/1ccd5a37604a48c781e8621168a4d6a0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/0edd1967783e45c6b87d4aa724f1d8a4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3eed28c9af3749679933fbf9e458aafa, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/23a8a2ed65ff409cb126430490900fd9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/ff30a3c9c81c4e00a36d6ab8a2410e69, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/976150b6156c4924af2922120e31d3f4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/d422a238e6bd4f8ba3159313f21a0739, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/62baaa59027f4f2badb1e3bb2676e400, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/a45fef5dedb4457bab74054fed1daa10] to archive 2024-12-08T08:16:04,919 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:16:04,920 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3ac37d2cc3fa4c40ae79a9bbb1833e8a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3ac37d2cc3fa4c40ae79a9bbb1833e8a 2024-12-08T08:16:04,921 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/25554c7f61d84f52ba3e521f42d0c064 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/25554c7f61d84f52ba3e521f42d0c064 2024-12-08T08:16:04,922 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/534cd88268cc44caa4764ff749231827 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/534cd88268cc44caa4764ff749231827 2024-12-08T08:16:04,923 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/ae4e55bdd8bf44bd80158efdd3f792fa to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/ae4e55bdd8bf44bd80158efdd3f792fa 2024-12-08T08:16:04,924 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/88e12fbb11664b68878266b31d73571d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/88e12fbb11664b68878266b31d73571d 2024-12-08T08:16:04,925 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/2c47114679c042d8916a27e7eaa82578 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/2c47114679c042d8916a27e7eaa82578 2024-12-08T08:16:04,926 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/aa52c08b730a4365a43c5e70229dc0a2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/aa52c08b730a4365a43c5e70229dc0a2 2024-12-08T08:16:04,927 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/0e06da802d814bc5aa2a62a38da89d82 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/0e06da802d814bc5aa2a62a38da89d82 2024-12-08T08:16:04,928 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3cee63bfcd9845c78a7d10c60c23e275 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3cee63bfcd9845c78a7d10c60c23e275 2024-12-08T08:16:04,929 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/a63a85ca96814e78a7dc14815c6290fa to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/a63a85ca96814e78a7dc14815c6290fa 2024-12-08T08:16:04,931 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/e017d72fbaa849fbb7d813a21c5df195 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/e017d72fbaa849fbb7d813a21c5df195 2024-12-08T08:16:04,932 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/c0d576700c14452384576ef1c6b85e35 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/c0d576700c14452384576ef1c6b85e35 2024-12-08T08:16:04,933 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/1ccd5a37604a48c781e8621168a4d6a0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/1ccd5a37604a48c781e8621168a4d6a0 2024-12-08T08:16:04,934 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/0edd1967783e45c6b87d4aa724f1d8a4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/0edd1967783e45c6b87d4aa724f1d8a4 2024-12-08T08:16:04,935 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3eed28c9af3749679933fbf9e458aafa to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/3eed28c9af3749679933fbf9e458aafa 2024-12-08T08:16:04,936 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/23a8a2ed65ff409cb126430490900fd9 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/23a8a2ed65ff409cb126430490900fd9 2024-12-08T08:16:04,951 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/ff30a3c9c81c4e00a36d6ab8a2410e69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/ff30a3c9c81c4e00a36d6ab8a2410e69 2024-12-08T08:16:04,952 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/976150b6156c4924af2922120e31d3f4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/976150b6156c4924af2922120e31d3f4 2024-12-08T08:16:04,953 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/d422a238e6bd4f8ba3159313f21a0739 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/d422a238e6bd4f8ba3159313f21a0739 2024-12-08T08:16:04,954 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/62baaa59027f4f2badb1e3bb2676e400 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/62baaa59027f4f2badb1e3bb2676e400 2024-12-08T08:16:04,955 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/a45fef5dedb4457bab74054fed1daa10 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/a45fef5dedb4457bab74054fed1daa10 2024-12-08T08:16:04,958 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/90c13e228efc4aafa66e6fb4aa713dbe, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/abd6b213aa714cee9179af26f39a72f3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/2b0d27e1663a4a74ab18353fa50b6538, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/08c87666999047cf933eddd7b1884c77, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/141cca72f4dd48e98f383091eb331032, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/cd2f5b43c318461bac2ed4dd2de24fd8, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/86cbe3ea52cd43928e74c28a214cd570, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/70a79e46fe8a4a7fbcaee93128b049d7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/d5261a7fd4ee436fb2206bf65d966fe3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/414bf55d101748729dd6ef8e4b5a47ad, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/1d4737b6120144acb8d1d08d73477584, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/cb1ed595b67e4e41848355ef3cd16c01, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/c5ea226058724dd383cc56e15145aa3c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/1d93c3814930481c874e59d4f979f4af, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/b06c0191c6a94e68be973c56d4e7a170, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/9fabc11432a04fa6ba056e99b1019341, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/7c5663cae18c4f888a8f256aa0a76cd3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/a937d4271e534a499210cd853f9e9f7a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/464d9175f4c94613a2ccf8560f3b2fce, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/721b7028fc61446ea0eb2fe0713e42c3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/9e0027e1d9464ed3b3b8062f02bb210c] to archive 2024-12-08T08:16:04,959 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:16:04,960 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/90c13e228efc4aafa66e6fb4aa713dbe to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/90c13e228efc4aafa66e6fb4aa713dbe 2024-12-08T08:16:04,961 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/abd6b213aa714cee9179af26f39a72f3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/abd6b213aa714cee9179af26f39a72f3 2024-12-08T08:16:04,961 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/2b0d27e1663a4a74ab18353fa50b6538 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/2b0d27e1663a4a74ab18353fa50b6538 2024-12-08T08:16:04,962 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/08c87666999047cf933eddd7b1884c77 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/08c87666999047cf933eddd7b1884c77 2024-12-08T08:16:04,963 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/141cca72f4dd48e98f383091eb331032 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/141cca72f4dd48e98f383091eb331032 2024-12-08T08:16:04,964 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/cd2f5b43c318461bac2ed4dd2de24fd8 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/cd2f5b43c318461bac2ed4dd2de24fd8 2024-12-08T08:16:04,964 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/86cbe3ea52cd43928e74c28a214cd570 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/86cbe3ea52cd43928e74c28a214cd570 2024-12-08T08:16:04,965 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/70a79e46fe8a4a7fbcaee93128b049d7 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/70a79e46fe8a4a7fbcaee93128b049d7 2024-12-08T08:16:04,966 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/d5261a7fd4ee436fb2206bf65d966fe3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/d5261a7fd4ee436fb2206bf65d966fe3 2024-12-08T08:16:04,967 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/414bf55d101748729dd6ef8e4b5a47ad to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/414bf55d101748729dd6ef8e4b5a47ad 2024-12-08T08:16:04,967 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/1d4737b6120144acb8d1d08d73477584 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/1d4737b6120144acb8d1d08d73477584 2024-12-08T08:16:04,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/cb1ed595b67e4e41848355ef3cd16c01 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/cb1ed595b67e4e41848355ef3cd16c01 2024-12-08T08:16:04,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/c5ea226058724dd383cc56e15145aa3c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/c5ea226058724dd383cc56e15145aa3c 2024-12-08T08:16:04,969 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/1d93c3814930481c874e59d4f979f4af to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/1d93c3814930481c874e59d4f979f4af 2024-12-08T08:16:04,970 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/b06c0191c6a94e68be973c56d4e7a170 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/b06c0191c6a94e68be973c56d4e7a170 2024-12-08T08:16:04,971 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/9fabc11432a04fa6ba056e99b1019341 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/9fabc11432a04fa6ba056e99b1019341 2024-12-08T08:16:04,972 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/7c5663cae18c4f888a8f256aa0a76cd3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/7c5663cae18c4f888a8f256aa0a76cd3 2024-12-08T08:16:04,973 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/a937d4271e534a499210cd853f9e9f7a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/a937d4271e534a499210cd853f9e9f7a 2024-12-08T08:16:04,973 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/464d9175f4c94613a2ccf8560f3b2fce to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/464d9175f4c94613a2ccf8560f3b2fce 2024-12-08T08:16:04,974 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/721b7028fc61446ea0eb2fe0713e42c3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/721b7028fc61446ea0eb2fe0713e42c3 2024-12-08T08:16:04,975 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/9e0027e1d9464ed3b3b8062f02bb210c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/9e0027e1d9464ed3b3b8062f02bb210c 2024-12-08T08:16:07,462 INFO [Thread-1605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1605. 2024-12-08T08:16:07,462 DEBUG [Thread-1605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:16:07,463 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:07,463 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:07,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 155 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1455 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4360 rows 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1442 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4321 rows 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1449 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4342 rows 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1451 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4348 rows 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1457 2024-12-08T08:16:07,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4369 rows 2024-12-08T08:16:07,463 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T08:16:07,463 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T08:16:07,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:07,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:07,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:16:07,464 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-08T08:16:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T08:16:07,467 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645767467"}]},"ts":"1733645767467"} 2024-12-08T08:16:07,468 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T08:16:07,468 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T08:16:07,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T08:16:07,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, UNASSIGN}] 2024-12-08T08:16:07,470 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, UNASSIGN 2024-12-08T08:16:07,471 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=702b91089e160458511de9d558436b69, regionState=CLOSING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:16:07,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, UNASSIGN because future has completed 2024-12-08T08:16:07,472 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T08:16:07,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:16:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T08:16:07,625 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 702b91089e160458511de9d558436b69 2024-12-08T08:16:07,625 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T08:16:07,625 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 702b91089e160458511de9d558436b69, disabling compactions & flushes 2024-12-08T08:16:07,625 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:16:07,625 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:16:07,625 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. after waiting 0 ms 2024-12-08T08:16:07,625 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:16:07,625 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 702b91089e160458511de9d558436b69 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-08T08:16:07,625 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=A 2024-12-08T08:16:07,625 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:07,626 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=B 2024-12-08T08:16:07,626 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:07,626 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 702b91089e160458511de9d558436b69, store=C 2024-12-08T08:16:07,626 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:07,631 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208207bce5d59494946919eeacfe03134eb_702b91089e160458511de9d558436b69 is 50, key is test_row_2/A:col10/1733645767461/Put/seqid=0 2024-12-08T08:16:07,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742269_1445 (size=7374) 2024-12-08T08:16:07,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T08:16:08,036 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:08,040 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208207bce5d59494946919eeacfe03134eb_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208207bce5d59494946919eeacfe03134eb_702b91089e160458511de9d558436b69 2024-12-08T08:16:08,041 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/57c24e719e224ae6be002ecd00f50ec0, store: [table=TestAcidGuarantees family=A region=702b91089e160458511de9d558436b69] 2024-12-08T08:16:08,041 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/57c24e719e224ae6be002ecd00f50ec0 is 175, key is test_row_2/A:col10/1733645767461/Put/seqid=0 2024-12-08T08:16:08,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742270_1446 (size=13865) 2024-12-08T08:16:08,045 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=443, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/57c24e719e224ae6be002ecd00f50ec0 2024-12-08T08:16:08,052 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/9aabad02634343a5b6541e8d1e6d9b17 is 50, key is test_row_2/B:col10/1733645767461/Put/seqid=0 2024-12-08T08:16:08,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742271_1447 (size=7415) 2024-12-08T08:16:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T08:16:08,456 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/9aabad02634343a5b6541e8d1e6d9b17 2024-12-08T08:16:08,461 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/c4dc1f5973b544f78d0eadac81d8c742 is 50, key is test_row_2/C:col10/1733645767461/Put/seqid=0 2024-12-08T08:16:08,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742272_1448 (size=7415) 2024-12-08T08:16:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T08:16:08,865 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/c4dc1f5973b544f78d0eadac81d8c742 2024-12-08T08:16:08,869 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/A/57c24e719e224ae6be002ecd00f50ec0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/57c24e719e224ae6be002ecd00f50ec0 2024-12-08T08:16:08,873 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/57c24e719e224ae6be002ecd00f50ec0, entries=50, sequenceid=443, filesize=13.5 K 2024-12-08T08:16:08,874 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/B/9aabad02634343a5b6541e8d1e6d9b17 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/9aabad02634343a5b6541e8d1e6d9b17 2024-12-08T08:16:08,879 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/9aabad02634343a5b6541e8d1e6d9b17, entries=50, sequenceid=443, filesize=7.2 K 2024-12-08T08:16:08,880 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/.tmp/C/c4dc1f5973b544f78d0eadac81d8c742 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/c4dc1f5973b544f78d0eadac81d8c742 2024-12-08T08:16:08,884 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/c4dc1f5973b544f78d0eadac81d8c742, entries=50, sequenceid=443, filesize=7.2 K 2024-12-08T08:16:08,885 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 702b91089e160458511de9d558436b69 in 1260ms, sequenceid=443, compaction requested=true 2024-12-08T08:16:08,890 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/recovered.edits/446.seqid, newMaxSeqId=446, maxSeqId=4 2024-12-08T08:16:08,891 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. 2024-12-08T08:16:08,891 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 702b91089e160458511de9d558436b69: Waiting for close lock at 1733645767625Running coprocessor pre-close hooks at 1733645767625Disabling compacts and flushes for region at 1733645767625Disabling writes for close at 1733645767625Obtaining lock to block concurrent updates at 1733645767625Preparing flush snapshotting stores in 702b91089e160458511de9d558436b69 at 1733645767625Finished memstore snapshotting TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733645767626 (+1 ms)Flushing stores of TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69. at 1733645767626Flushing 702b91089e160458511de9d558436b69/A: creating writer at 1733645767626Flushing mob file 702b91089e160458511de9d558436b69/A: appending metadata at 1733645767631 (+5 ms)Flushing mob file 702b91089e160458511de9d558436b69/A: closing flushed file at 1733645767631Flushing 702b91089e160458511de9d558436b69/A: appending metadata at 1733645768041 (+410 ms)Flushing 702b91089e160458511de9d558436b69/A: closing flushed file at 1733645768041Flushing 702b91089e160458511de9d558436b69/B: creating writer at 1733645768049 (+8 ms)Flushing 702b91089e160458511de9d558436b69/B: appending metadata at 1733645768051 (+2 ms)Flushing 702b91089e160458511de9d558436b69/B: closing flushed file at 1733645768051Flushing 702b91089e160458511de9d558436b69/C: creating writer at 1733645768459 (+408 ms)Flushing 702b91089e160458511de9d558436b69/C: appending metadata at 1733645768461 (+2 ms)Flushing 702b91089e160458511de9d558436b69/C: closing flushed file at 1733645768461Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@444a1b7d: reopening flushed file at 1733645768869 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@698674aa: reopening flushed file at 1733645768874 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60e7ac64: reopening flushed file at 1733645768879 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 702b91089e160458511de9d558436b69 in 1260ms, sequenceid=443, compaction requested=true at 1733645768885 (+6 ms)Writing region close event to WAL at 1733645768887 (+2 ms)Running coprocessor post-close hooks at 1733645768891 (+4 ms)Closed at 1733645768891 2024-12-08T08:16:08,893 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 702b91089e160458511de9d558436b69 2024-12-08T08:16:08,893 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=702b91089e160458511de9d558436b69, regionState=CLOSED 2024-12-08T08:16:08,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:16:08,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-08T08:16:08,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 702b91089e160458511de9d558436b69, server=422e41b75614,38619,1733645641981 in 1.4230 sec 2024-12-08T08:16:08,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-08T08:16:08,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=702b91089e160458511de9d558436b69, UNASSIGN in 1.4270 sec 2024-12-08T08:16:08,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-08T08:16:08,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4310 sec 2024-12-08T08:16:08,901 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645768901"}]},"ts":"1733645768901"} 2024-12-08T08:16:08,902 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T08:16:08,902 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T08:16:08,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4390 sec 2024-12-08T08:16:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T08:16:09,611 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:09,611 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-08T08:16:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:09,612 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T08:16:09,613 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:09,614 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69 2024-12-08T08:16:09,616 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/recovered.edits] 2024-12-08T08:16:09,618 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/57c24e719e224ae6be002ecd00f50ec0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/57c24e719e224ae6be002ecd00f50ec0 2024-12-08T08:16:09,618 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/66398e8fcfca4cd7b787734840281808 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/66398e8fcfca4cd7b787734840281808 2024-12-08T08:16:09,619 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/9153811e543b4d1ab8e775e871aae4f9 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/A/9153811e543b4d1ab8e775e871aae4f9 2024-12-08T08:16:09,621 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/2cdc8c52eec74cb787414f8d4184a35e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/2cdc8c52eec74cb787414f8d4184a35e 2024-12-08T08:16:09,622 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/393cf5b5d97d49ec91465b9e3f42096f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/393cf5b5d97d49ec91465b9e3f42096f 2024-12-08T08:16:09,623 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/9aabad02634343a5b6541e8d1e6d9b17 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/B/9aabad02634343a5b6541e8d1e6d9b17 2024-12-08T08:16:09,624 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/899b78fa180e4a7795d0cabd6eed07cf to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/899b78fa180e4a7795d0cabd6eed07cf 2024-12-08T08:16:09,625 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/c4dc1f5973b544f78d0eadac81d8c742 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/c4dc1f5973b544f78d0eadac81d8c742 2024-12-08T08:16:09,626 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/db6d86db12b446f0bb8d38e236bf32a3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/C/db6d86db12b446f0bb8d38e236bf32a3 2024-12-08T08:16:09,628 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/recovered.edits/446.seqid to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69/recovered.edits/446.seqid 2024-12-08T08:16:09,628 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/702b91089e160458511de9d558436b69 2024-12-08T08:16:09,628 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T08:16:09,628 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T08:16:09,629 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T08:16:09,631 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120809dc4e9effe840729f87ce2f4e5235aa_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120809dc4e9effe840729f87ce2f4e5235aa_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,632 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080dcaa653cf414002b8b9234660bcd93a_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080dcaa653cf414002b8b9234660bcd93a_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,632 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208207bce5d59494946919eeacfe03134eb_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208207bce5d59494946919eeacfe03134eb_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,633 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120825ed69c5549f43a983c60a137a6fc2c0_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120825ed69c5549f43a983c60a137a6fc2c0_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,634 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208277ecb747eb949a4bc39134980c35824_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208277ecb747eb949a4bc39134980c35824_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,635 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208309bca6d4e5749c681c76f0e78f45b2a_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208309bca6d4e5749c681c76f0e78f45b2a_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,636 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208334ae6a58f9f4ddbbf92d15e344e5330_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208334ae6a58f9f4ddbbf92d15e344e5330_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,637 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083d30e04b523c4672b5da99a9486d615c_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083d30e04b523c4672b5da99a9486d615c_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,637 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120841f5962a7ead489899f6f3a9555fa3fa_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120841f5962a7ead489899f6f3a9555fa3fa_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,638 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208550ee05e6f944efc8af898527e1144a8_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208550ee05e6f944efc8af898527e1144a8_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,639 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120870157324ba194ab9b4e48bf7bad11f50_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120870157324ba194ab9b4e48bf7bad11f50_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,640 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087bb5deef698d421e9180143a27b178b0_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087bb5deef698d421e9180143a27b178b0_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,640 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088f098df96d4a487c9e891d469d7bfcac_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088f098df96d4a487c9e891d469d7bfcac_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,641 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089ea2747155274c59b145df0644f0e35f_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089ea2747155274c59b145df0644f0e35f_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,642 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a0477aba97c5445ebfdf528b345adfdb_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a0477aba97c5445ebfdf528b345adfdb_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,643 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208cfe0dd7d21f44ee3a7f36c86a2101c98_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208cfe0dd7d21f44ee3a7f36c86a2101c98_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,643 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dc9c0ac8380243aa9fb79d38626d0762_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dc9c0ac8380243aa9fb79d38626d0762_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,644 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f658424bd6164826aa62b0bf78f87cce_702b91089e160458511de9d558436b69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f658424bd6164826aa62b0bf78f87cce_702b91089e160458511de9d558436b69 2024-12-08T08:16:09,645 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T08:16:09,646 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:09,648 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T08:16:09,649 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T08:16:09,651 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:09,651 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T08:16:09,651 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733645769651"}]},"ts":"9223372036854775807"} 2024-12-08T08:16:09,653 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T08:16:09,653 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 702b91089e160458511de9d558436b69, NAME => 'TestAcidGuarantees,,1733645734654.702b91089e160458511de9d558436b69.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T08:16:09,653 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T08:16:09,653 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733645769653"}]},"ts":"9223372036854775807"} 2024-12-08T08:16:09,654 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T08:16:09,655 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:09,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-12-08T08:16:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T08:16:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T08:16:09,721 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:09,728 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 252), OpenFileDescriptor=445 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=404 (was 417), ProcessCount=11 (was 11), AvailableMemoryMB=7466 (was 6525) - AvailableMemoryMB LEAK? - 2024-12-08T08:16:09,735 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=404, ProcessCount=11, AvailableMemoryMB=7466 2024-12-08T08:16:09,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T08:16:09,737 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:16:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:09,739 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:16:09,739 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:09,739 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-12-08T08:16:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-08T08:16:09,740 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:16:09,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742273_1449 (size=1061) 2024-12-08T08:16:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-08T08:16:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-08T08:16:10,148 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b 2024-12-08T08:16:10,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742274_1450 (size=53) 2024-12-08T08:16:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-08T08:16:10,560 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:16:10,560 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 92943a79353f8896400eee756d87f758, disabling compactions & flushes 2024-12-08T08:16:10,560 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:10,560 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:10,560 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. after waiting 0 ms 2024-12-08T08:16:10,560 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:10,560 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:10,560 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 92943a79353f8896400eee756d87f758: Waiting for close lock at 1733645770560Disabling compacts and flushes for region at 1733645770560Disabling writes for close at 1733645770560Writing region close event to WAL at 1733645770560Closed at 1733645770560 2024-12-08T08:16:10,561 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:16:10,562 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733645770562"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733645770562"}]},"ts":"1733645770562"} 2024-12-08T08:16:10,564 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:16:10,565 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:16:10,565 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645770565"}]},"ts":"1733645770565"} 2024-12-08T08:16:10,567 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T08:16:10,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92943a79353f8896400eee756d87f758, ASSIGN}] 2024-12-08T08:16:10,569 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92943a79353f8896400eee756d87f758, ASSIGN 2024-12-08T08:16:10,570 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=92943a79353f8896400eee756d87f758, ASSIGN; state=OFFLINE, location=422e41b75614,38619,1733645641981; forceNewPlan=false, retain=false 2024-12-08T08:16:10,721 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=92943a79353f8896400eee756d87f758, regionState=OPENING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:16:10,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=92943a79353f8896400eee756d87f758, ASSIGN because future has completed 2024-12-08T08:16:10,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:16:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-08T08:16:10,881 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:10,881 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:16:10,882 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 92943a79353f8896400eee756d87f758 2024-12-08T08:16:10,882 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:16:10,882 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 92943a79353f8896400eee756d87f758 2024-12-08T08:16:10,882 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 92943a79353f8896400eee756d87f758 2024-12-08T08:16:10,884 INFO [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 92943a79353f8896400eee756d87f758 2024-12-08T08:16:10,885 INFO [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:16:10,885 INFO [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92943a79353f8896400eee756d87f758 columnFamilyName A 2024-12-08T08:16:10,885 DEBUG [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:10,886 INFO [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] regionserver.HStore(327): Store=92943a79353f8896400eee756d87f758/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:16:10,886 INFO [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 92943a79353f8896400eee756d87f758 2024-12-08T08:16:10,887 INFO [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:16:10,887 INFO [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92943a79353f8896400eee756d87f758 columnFamilyName B 2024-12-08T08:16:10,887 DEBUG [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:10,888 INFO [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] regionserver.HStore(327): Store=92943a79353f8896400eee756d87f758/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:16:10,888 INFO [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 92943a79353f8896400eee756d87f758 2024-12-08T08:16:10,889 INFO [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:16:10,889 INFO [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92943a79353f8896400eee756d87f758 columnFamilyName C 2024-12-08T08:16:10,889 DEBUG [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:10,890 INFO [StoreOpener-92943a79353f8896400eee756d87f758-1 {}] regionserver.HStore(327): Store=92943a79353f8896400eee756d87f758/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:16:10,890 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:10,890 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 92943a79353f8896400eee756d87f758 2024-12-08T08:16:10,891 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758 2024-12-08T08:16:10,892 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758 2024-12-08T08:16:10,893 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 92943a79353f8896400eee756d87f758 2024-12-08T08:16:10,893 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 92943a79353f8896400eee756d87f758 2024-12-08T08:16:10,894 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T08:16:10,895 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 92943a79353f8896400eee756d87f758 2024-12-08T08:16:10,898 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:16:10,898 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 92943a79353f8896400eee756d87f758; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68219985, jitterRate=0.0165569931268692}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T08:16:10,898 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 92943a79353f8896400eee756d87f758 2024-12-08T08:16:10,899 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 92943a79353f8896400eee756d87f758: Running coprocessor pre-open hook at 1733645770882Writing region info on filesystem at 1733645770882Initializing all the Stores at 1733645770883 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645770883Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645770884 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645770884Cleaning up temporary data from old regions at 1733645770893 (+9 ms)Running coprocessor post-open hooks at 1733645770898 (+5 ms)Region opened successfully at 1733645770899 (+1 ms) 2024-12-08T08:16:10,900 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., pid=118, masterSystemTime=1733645770878 2024-12-08T08:16:10,902 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:10,902 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:10,903 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=92943a79353f8896400eee756d87f758, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:16:10,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:16:10,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-08T08:16:10,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 in 182 msec 2024-12-08T08:16:10,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-12-08T08:16:10,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92943a79353f8896400eee756d87f758, ASSIGN in 340 msec 2024-12-08T08:16:10,909 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:16:10,910 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645770909"}]},"ts":"1733645770909"} 2024-12-08T08:16:10,911 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T08:16:10,911 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:16:10,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-12-08T08:16:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-08T08:16:11,892 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:11,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ded0f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:11,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:11,895 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd2060e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:11,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:11,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:11,898 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:11,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d26bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,900 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:11,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:11,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:11,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:11,903 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T08:16:11,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 422e41b75614,44803,1733645641278 2024-12-08T08:16:11,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5a7eb3ab 2024-12-08T08:16:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T08:16:11,905 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T08:16:11,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:11,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:11,908 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6839617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:11,910 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1727b097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:11,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:11,912 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b83d6ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:11,913 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46853c01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:11,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:11,918 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bae295, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:11,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:11,919 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fba04f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:11,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:11,922 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af3dbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:11,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:11,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e1da258, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:11,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:11,925 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:11,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:11,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4414a158, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:11,926 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37db0915, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:11,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:11,928 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:11,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:11,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:11,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e8ed41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:11,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:11,929 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13286f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:11,930 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:11,931 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48487887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:11,932 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@189b4cd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:11,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:11,933 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ca73de3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:11,934 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:11,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d0a394, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,935 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:11,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:11,936 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29d27fea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:11,937 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23fddbe5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:11,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:11,938 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:11,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:11,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:11,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67fbad18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:11,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:11,939 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0f361a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:11,940 DEBUG [Thread-2038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:11,940 DEBUG [Thread-2023 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:11,940 DEBUG [Thread-2034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:11,940 DEBUG [Thread-2025 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:11,941 DEBUG [Thread-2027 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:11,941 DEBUG [Thread-2021 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:11,941 DEBUG [Thread-2032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:11,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:11,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:11,942 DEBUG [Thread-2040 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:11,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:11,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:11,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:11,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:11,944 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:16:11,944 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:16:11,944 DEBUG [Thread-2029 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:16:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:16:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:16:11,947 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:16:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:16:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:16:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-08T08:16:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:11,948 DEBUG [Thread-2036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:11,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T08:16:11,949 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:11,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:11,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:11,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:11,950 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:11,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:16:11,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:11,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T08:16:11,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2] 2024-12-08T08:16:11,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:11,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:11,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:11,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:11,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:11,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:11,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/57ac564612714ada9efe9c44ce4c743b is 50, key is test_row_0/A:col10/1733645771955/Put/seqid=0 2024-12-08T08:16:11,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645831979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:11,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742275_1451 (size=12001) 2024-12-08T08:16:11,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645831980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:11,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645831981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:11,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:11,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645831983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:11,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645831986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:11,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:11,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:11,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:11,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T08:16:12,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645832091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645832091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645832091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645832100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645832101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T08:16:12,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:12,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:12,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:12,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T08:16:12,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:12,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:12,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:12,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T08:16:12,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645832300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645832300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645832300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645832310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645832310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/57ac564612714ada9efe9c44ce4c743b 2024-12-08T08:16:12,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/36ebea7146664b229b9af8afe018128b is 50, key is test_row_0/B:col10/1733645771955/Put/seqid=0 2024-12-08T08:16:12,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742276_1452 (size=12001) 2024-12-08T08:16:12,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T08:16:12,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:12,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:12,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:12,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T08:16:12,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:12,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:12,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:12,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T08:16:12,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645832610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645832610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645832610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645832621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645832621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:12,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:12,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T08:16:12,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:12,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:12,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:12,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/36ebea7146664b229b9af8afe018128b 2024-12-08T08:16:12,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/e9e56dda1981448e9c80f61befe5954e is 50, key is test_row_0/C:col10/1733645771955/Put/seqid=0 2024-12-08T08:16:12,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742277_1453 (size=12001) 2024-12-08T08:16:12,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T08:16:12,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:12,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:12,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:12,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:13,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T08:16:13,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:13,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:13,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T08:16:13,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:13,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645833120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:13,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645833131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:13,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:13,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645833131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:13,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:13,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645833131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:13,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:13,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645833140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:13,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T08:16:13,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:13,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:13,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:13,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:13,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:13,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/e9e56dda1981448e9c80f61befe5954e 2024-12-08T08:16:13,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/57ac564612714ada9efe9c44ce4c743b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/57ac564612714ada9efe9c44ce4c743b 2024-12-08T08:16:13,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/57ac564612714ada9efe9c44ce4c743b, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T08:16:13,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/36ebea7146664b229b9af8afe018128b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/36ebea7146664b229b9af8afe018128b 2024-12-08T08:16:13,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/36ebea7146664b229b9af8afe018128b, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T08:16:13,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/e9e56dda1981448e9c80f61befe5954e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e9e56dda1981448e9c80f61befe5954e 2024-12-08T08:16:13,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e9e56dda1981448e9c80f61befe5954e, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T08:16:13,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 92943a79353f8896400eee756d87f758 in 1301ms, sequenceid=13, compaction requested=false 2024-12-08T08:16:13,258 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T08:16:13,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:13,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T08:16:13,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:13,332 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T08:16:13,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:13,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:13,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:13,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:13,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:13,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:13,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/6e8d0134c8c0451f891cebcedbd47298 is 50, key is test_row_0/A:col10/1733645771969/Put/seqid=0 2024-12-08T08:16:13,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742278_1454 (size=12001) 2024-12-08T08:16:13,381 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/6e8d0134c8c0451f891cebcedbd47298 2024-12-08T08:16:13,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/c7f000c4c2364c98a512683eaf5215f1 is 50, key is test_row_0/B:col10/1733645771969/Put/seqid=0 2024-12-08T08:16:13,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742279_1455 (size=12001) 2024-12-08T08:16:13,427 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/c7f000c4c2364c98a512683eaf5215f1 2024-12-08T08:16:13,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/f6c63d8cda814ea4b6765d601a78d611 is 50, key is test_row_0/C:col10/1733645771969/Put/seqid=0 2024-12-08T08:16:13,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742280_1456 (size=12001) 2024-12-08T08:16:13,868 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/f6c63d8cda814ea4b6765d601a78d611 2024-12-08T08:16:13,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/6e8d0134c8c0451f891cebcedbd47298 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/6e8d0134c8c0451f891cebcedbd47298 2024-12-08T08:16:13,886 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/6e8d0134c8c0451f891cebcedbd47298, entries=150, sequenceid=33, filesize=11.7 K 2024-12-08T08:16:13,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/c7f000c4c2364c98a512683eaf5215f1 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c7f000c4c2364c98a512683eaf5215f1 2024-12-08T08:16:13,892 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c7f000c4c2364c98a512683eaf5215f1, entries=150, sequenceid=33, filesize=11.7 K 2024-12-08T08:16:13,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/f6c63d8cda814ea4b6765d601a78d611 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/f6c63d8cda814ea4b6765d601a78d611 2024-12-08T08:16:13,898 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/f6c63d8cda814ea4b6765d601a78d611, entries=150, sequenceid=33, filesize=11.7 K 2024-12-08T08:16:13,900 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 92943a79353f8896400eee756d87f758 in 568ms, sequenceid=33, compaction requested=false 2024-12-08T08:16:13,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:13,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:13,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-08T08:16:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-08T08:16:13,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-08T08:16:13,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9530 sec 2024-12-08T08:16:13,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.9630 sec 2024-12-08T08:16:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T08:16:14,102 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:14,102 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-08T08:16:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T08:16:14,105 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:14,107 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:14,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:14,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:16:14,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:14,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:14,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:14,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:14,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:14,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:14,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/42ad33b596044da6acdf5c6b43ebb4d6 is 50, key is test_row_0/A:col10/1733645774154/Put/seqid=0 2024-12-08T08:16:14,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742281_1457 (size=14341) 2024-12-08T08:16:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T08:16:14,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645774174, max timestamp=1733645774222Num uniques 859; 2024-12-08T08:16:14,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:16:14,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645774174, max timestamp=1733645774222Num uniques 859; 2024-12-08T08:16:14,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:16:14,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645774174, max timestamp=1733645774222Num uniques 859; 2024-12-08T08:16:14,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:16:14,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T08:16:14,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:14,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:14,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:14,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645834267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645834269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645834270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645834272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645834288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645834390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645834401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645834401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645834401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T08:16:14,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:14,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:14,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:14,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T08:16:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645834421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,439 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T08:16:14,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T08:16:14,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:14,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:14,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:14,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645834601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/42ad33b596044da6acdf5c6b43ebb4d6 2024-12-08T08:16:14,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/c33555425c174babb37c053640bac70e is 50, key is test_row_0/B:col10/1733645774154/Put/seqid=0 2024-12-08T08:16:14,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742282_1458 (size=12001) 2024-12-08T08:16:14,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645834620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645834621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645834621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645834641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T08:16:14,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:14,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:14,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:14,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T08:16:14,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T08:16:14,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:14,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:14,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:14,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:14,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645834911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645834940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645834941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645834941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:14,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645834961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:15,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/c33555425c174babb37c053640bac70e 2024-12-08T08:16:15,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T08:16:15,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:15,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:15,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:15,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:15,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:15,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/e74a1a5dc8764bd8997923e274d50928 is 50, key is test_row_0/C:col10/1733645774154/Put/seqid=0 2024-12-08T08:16:15,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742283_1459 (size=12001) 2024-12-08T08:16:15,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/e74a1a5dc8764bd8997923e274d50928 2024-12-08T08:16:15,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/42ad33b596044da6acdf5c6b43ebb4d6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/42ad33b596044da6acdf5c6b43ebb4d6 2024-12-08T08:16:15,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/42ad33b596044da6acdf5c6b43ebb4d6, entries=200, sequenceid=43, filesize=14.0 K 2024-12-08T08:16:15,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/c33555425c174babb37c053640bac70e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c33555425c174babb37c053640bac70e 2024-12-08T08:16:15,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c33555425c174babb37c053640bac70e, entries=150, sequenceid=43, filesize=11.7 K 2024-12-08T08:16:15,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/e74a1a5dc8764bd8997923e274d50928 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e74a1a5dc8764bd8997923e274d50928 2024-12-08T08:16:15,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e74a1a5dc8764bd8997923e274d50928, entries=150, sequenceid=43, filesize=11.7 K 2024-12-08T08:16:15,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 92943a79353f8896400eee756d87f758 in 896ms, sequenceid=43, compaction requested=true 2024-12-08T08:16:15,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:15,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:15,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:15,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:15,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:15,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:15,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:15,054 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:15,054 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:15,054 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:15,054 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:15,054 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/B is initiating minor compaction (all files) 2024-12-08T08:16:15,054 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/A is initiating minor compaction (all files) 2024-12-08T08:16:15,054 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/A in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:15,054 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/B in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:15,055 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/57ac564612714ada9efe9c44ce4c743b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/6e8d0134c8c0451f891cebcedbd47298, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/42ad33b596044da6acdf5c6b43ebb4d6] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=37.4 K 2024-12-08T08:16:15,055 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/36ebea7146664b229b9af8afe018128b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c7f000c4c2364c98a512683eaf5215f1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c33555425c174babb37c053640bac70e] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=35.2 K 2024-12-08T08:16:15,055 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57ac564612714ada9efe9c44ce4c743b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733645771947 2024-12-08T08:16:15,055 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ebea7146664b229b9af8afe018128b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733645771947 2024-12-08T08:16:15,055 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e8d0134c8c0451f891cebcedbd47298, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733645771969 2024-12-08T08:16:15,055 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting c7f000c4c2364c98a512683eaf5215f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733645771969 2024-12-08T08:16:15,055 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42ad33b596044da6acdf5c6b43ebb4d6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733645774143 2024-12-08T08:16:15,055 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting c33555425c174babb37c053640bac70e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733645774153 2024-12-08T08:16:15,065 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#B#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:15,066 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/2e43bf1b698e4664a7be43f166d7898a is 50, key is test_row_0/B:col10/1733645774154/Put/seqid=0 2024-12-08T08:16:15,066 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#A#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:15,067 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/5b18bcf1249844109a6f954de971dbed is 50, key is test_row_0/A:col10/1733645774154/Put/seqid=0 2024-12-08T08:16:15,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742284_1460 (size=12104) 2024-12-08T08:16:15,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742285_1461 (size=12104) 2024-12-08T08:16:15,082 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/5b18bcf1249844109a6f954de971dbed as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5b18bcf1249844109a6f954de971dbed 2024-12-08T08:16:15,083 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/2e43bf1b698e4664a7be43f166d7898a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2e43bf1b698e4664a7be43f166d7898a 2024-12-08T08:16:15,096 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/A of 92943a79353f8896400eee756d87f758 into 5b18bcf1249844109a6f954de971dbed(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:15,096 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:15,096 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/A, priority=13, startTime=1733645775053; duration=0sec 2024-12-08T08:16:15,096 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:15,096 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:A 2024-12-08T08:16:15,096 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:15,098 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:15,098 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/C is initiating minor compaction (all files) 2024-12-08T08:16:15,098 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/C in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:15,098 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e9e56dda1981448e9c80f61befe5954e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/f6c63d8cda814ea4b6765d601a78d611, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e74a1a5dc8764bd8997923e274d50928] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=35.2 K 2024-12-08T08:16:15,098 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9e56dda1981448e9c80f61befe5954e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733645771947 2024-12-08T08:16:15,098 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6c63d8cda814ea4b6765d601a78d611, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733645771969 2024-12-08T08:16:15,099 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e74a1a5dc8764bd8997923e274d50928, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733645774153 2024-12-08T08:16:15,104 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/B of 92943a79353f8896400eee756d87f758 into 2e43bf1b698e4664a7be43f166d7898a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:15,104 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:15,104 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/B, priority=13, startTime=1733645775053; duration=0sec 2024-12-08T08:16:15,104 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:15,104 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:B 2024-12-08T08:16:15,106 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#C#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:15,107 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/1247e21b06c94cbcb0a397b33e17587c is 50, key is test_row_0/C:col10/1733645774154/Put/seqid=0 2024-12-08T08:16:15,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742286_1462 (size=12104) 2024-12-08T08:16:15,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T08:16:15,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:15,182 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-08T08:16:15,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:15,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:15,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:15,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:15,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:15,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/5412320783d64eda841ecb0c308cb90c is 50, key is test_row_0/A:col10/1733645774270/Put/seqid=0 2024-12-08T08:16:15,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742287_1463 (size=12001) 2024-12-08T08:16:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T08:16:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:15,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:15,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645835460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645835461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:15,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645835465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:15,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645835469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:15,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645835480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:15,518 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/1247e21b06c94cbcb0a397b33e17587c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/1247e21b06c94cbcb0a397b33e17587c 2024-12-08T08:16:15,522 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/C of 92943a79353f8896400eee756d87f758 into 1247e21b06c94cbcb0a397b33e17587c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:15,522 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:15,522 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/C, priority=13, startTime=1733645775054; duration=0sec 2024-12-08T08:16:15,522 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:15,522 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:C 2024-12-08T08:16:15,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645835581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:15,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645835581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:15,595 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/5412320783d64eda841ecb0c308cb90c 2024-12-08T08:16:15,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/1d655e7756ea4c2a93ab2bbc96e0519e is 50, key is test_row_0/B:col10/1733645774270/Put/seqid=0 2024-12-08T08:16:15,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742288_1464 (size=12001) 2024-12-08T08:16:15,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645835790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:15,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645835790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:16,011 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/1d655e7756ea4c2a93ab2bbc96e0519e 2024-12-08T08:16:16,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/94d26a656d9a4dbe9ac98d0bc91d542d is 50, key is test_row_0/C:col10/1733645774270/Put/seqid=0 2024-12-08T08:16:16,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742289_1465 (size=12001) 2024-12-08T08:16:16,041 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/94d26a656d9a4dbe9ac98d0bc91d542d 2024-12-08T08:16:16,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/5412320783d64eda841ecb0c308cb90c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5412320783d64eda841ecb0c308cb90c 2024-12-08T08:16:16,049 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5412320783d64eda841ecb0c308cb90c, entries=150, sequenceid=81, filesize=11.7 K 2024-12-08T08:16:16,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/1d655e7756ea4c2a93ab2bbc96e0519e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/1d655e7756ea4c2a93ab2bbc96e0519e 2024-12-08T08:16:16,055 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/1d655e7756ea4c2a93ab2bbc96e0519e, entries=150, sequenceid=81, filesize=11.7 K 2024-12-08T08:16:16,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/94d26a656d9a4dbe9ac98d0bc91d542d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/94d26a656d9a4dbe9ac98d0bc91d542d 2024-12-08T08:16:16,060 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/94d26a656d9a4dbe9ac98d0bc91d542d, entries=150, sequenceid=81, filesize=11.7 K 2024-12-08T08:16:16,062 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 92943a79353f8896400eee756d87f758 in 880ms, sequenceid=81, compaction requested=false 2024-12-08T08:16:16,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:16,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:16,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-08T08:16:16,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-08T08:16:16,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-08T08:16:16,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9550 sec 2024-12-08T08:16:16,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.9670 sec 2024-12-08T08:16:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:16,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T08:16:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:16,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:16,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:16,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:16,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:16,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/ff5e167972594eb8a822723339b12d10 is 50, key is test_row_0/A:col10/1733645775451/Put/seqid=0 2024-12-08T08:16:16,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742290_1466 (size=16681) 2024-12-08T08:16:16,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/ff5e167972594eb8a822723339b12d10 2024-12-08T08:16:16,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/47db4e546a68472d8e06e5d343caddf7 is 50, key is test_row_0/B:col10/1733645775451/Put/seqid=0 2024-12-08T08:16:16,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742291_1467 (size=12001) 2024-12-08T08:16:16,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/47db4e546a68472d8e06e5d343caddf7 2024-12-08T08:16:16,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/33cdd86b07564359834a6a7b413c6063 is 50, key is test_row_0/C:col10/1733645775451/Put/seqid=0 2024-12-08T08:16:16,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645836216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:16,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645836226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:16,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742292_1468 (size=12001) 2024-12-08T08:16:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T08:16:16,251 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:16,252 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-08T08:16:16,254 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:16,255 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:16,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T08:16:16,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645836330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:16,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:16,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645836341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:16,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T08:16:16,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T08:16:16,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:16,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:16,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:16,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:16,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:16,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645836481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:16,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645836481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:16,484 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 92943a79353f8896400eee756d87f758 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:16:16,485 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 92943a79353f8896400eee756d87f758 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:16:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:16,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645836490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:16,496 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 92943a79353f8896400eee756d87f758 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:16:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:16,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645836541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:16,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645836551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:16,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:16,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T08:16:16,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:16,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:16,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:16,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:16,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T08:16:16,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/33cdd86b07564359834a6a7b413c6063 2024-12-08T08:16:16,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/ff5e167972594eb8a822723339b12d10 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ff5e167972594eb8a822723339b12d10 2024-12-08T08:16:16,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ff5e167972594eb8a822723339b12d10, entries=250, sequenceid=94, filesize=16.3 K 2024-12-08T08:16:16,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/47db4e546a68472d8e06e5d343caddf7 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/47db4e546a68472d8e06e5d343caddf7 2024-12-08T08:16:16,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/47db4e546a68472d8e06e5d343caddf7, entries=150, sequenceid=94, filesize=11.7 K 2024-12-08T08:16:16,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/33cdd86b07564359834a6a7b413c6063 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/33cdd86b07564359834a6a7b413c6063 2024-12-08T08:16:16,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/33cdd86b07564359834a6a7b413c6063, entries=150, sequenceid=94, filesize=11.7 K 2024-12-08T08:16:16,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 92943a79353f8896400eee756d87f758 in 543ms, sequenceid=94, compaction requested=true 2024-12-08T08:16:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:16,660 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:16,660 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:16,661 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:16,661 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/A is initiating minor compaction (all files) 2024-12-08T08:16:16,661 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/A in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:16,661 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5b18bcf1249844109a6f954de971dbed, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5412320783d64eda841ecb0c308cb90c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ff5e167972594eb8a822723339b12d10] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=39.8 K 2024-12-08T08:16:16,661 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:16,661 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/B is initiating minor compaction (all files) 2024-12-08T08:16:16,661 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/B in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:16,661 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2e43bf1b698e4664a7be43f166d7898a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/1d655e7756ea4c2a93ab2bbc96e0519e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/47db4e546a68472d8e06e5d343caddf7] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=35.3 K 2024-12-08T08:16:16,661 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b18bcf1249844109a6f954de971dbed, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733645774153 2024-12-08T08:16:16,662 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e43bf1b698e4664a7be43f166d7898a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733645774153 2024-12-08T08:16:16,662 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5412320783d64eda841ecb0c308cb90c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733645774253 2024-12-08T08:16:16,662 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d655e7756ea4c2a93ab2bbc96e0519e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733645774253 2024-12-08T08:16:16,662 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff5e167972594eb8a822723339b12d10, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733645775451 2024-12-08T08:16:16,662 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 47db4e546a68472d8e06e5d343caddf7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733645775451 2024-12-08T08:16:16,668 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:16,669 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/2c94800bbf544a53984c9e262c93bf57 is 50, key is test_row_0/B:col10/1733645775451/Put/seqid=0 2024-12-08T08:16:16,674 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#A#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:16,674 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/8f235f25bcf24bfaa7ff9b8272ae21a6 is 50, key is test_row_0/A:col10/1733645775451/Put/seqid=0 2024-12-08T08:16:16,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742293_1469 (size=12207) 2024-12-08T08:16:16,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742294_1470 (size=12207) 2024-12-08T08:16:16,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T08:16:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:16,714 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T08:16:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/53e6105f068d405ba3f730995c55432b is 50, key is test_row_0/A:col10/1733645776214/Put/seqid=0 2024-12-08T08:16:16,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742295_1471 (size=12001) 2024-12-08T08:16:16,748 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/53e6105f068d405ba3f730995c55432b 2024-12-08T08:16:16,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/2877aa8e11a543e6a41ad996f3d0a45d is 50, key is test_row_0/B:col10/1733645776214/Put/seqid=0 2024-12-08T08:16:16,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742296_1472 (size=12001) 2024-12-08T08:16:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:16,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T08:16:16,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645836890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:16,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645836901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:17,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645837010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645837020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:17,085 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/2c94800bbf544a53984c9e262c93bf57 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2c94800bbf544a53984c9e262c93bf57 2024-12-08T08:16:17,094 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/B of 92943a79353f8896400eee756d87f758 into 2c94800bbf544a53984c9e262c93bf57(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:17,094 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:17,094 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/B, priority=13, startTime=1733645776660; duration=0sec 2024-12-08T08:16:17,094 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:17,094 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:B 2024-12-08T08:16:17,094 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:17,096 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:17,097 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/C is initiating minor compaction (all files) 2024-12-08T08:16:17,097 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/C in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:17,097 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/1247e21b06c94cbcb0a397b33e17587c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/94d26a656d9a4dbe9ac98d0bc91d542d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/33cdd86b07564359834a6a7b413c6063] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=35.3 K 2024-12-08T08:16:17,097 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 1247e21b06c94cbcb0a397b33e17587c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733645774153 2024-12-08T08:16:17,097 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/8f235f25bcf24bfaa7ff9b8272ae21a6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/8f235f25bcf24bfaa7ff9b8272ae21a6 2024-12-08T08:16:17,097 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 94d26a656d9a4dbe9ac98d0bc91d542d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733645774253 2024-12-08T08:16:17,098 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 33cdd86b07564359834a6a7b413c6063, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733645775451 2024-12-08T08:16:17,101 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/A of 92943a79353f8896400eee756d87f758 into 8f235f25bcf24bfaa7ff9b8272ae21a6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:17,101 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:17,101 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/A, priority=13, startTime=1733645776660; duration=0sec 2024-12-08T08:16:17,102 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:17,102 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:A 2024-12-08T08:16:17,106 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#C#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:17,106 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/5d15c66750e74275b1ec9c400a9cae76 is 50, key is test_row_0/C:col10/1733645775451/Put/seqid=0 2024-12-08T08:16:17,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742297_1473 (size=12207) 2024-12-08T08:16:17,135 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/5d15c66750e74275b1ec9c400a9cae76 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/5d15c66750e74275b1ec9c400a9cae76 2024-12-08T08:16:17,146 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/C of 92943a79353f8896400eee756d87f758 into 5d15c66750e74275b1ec9c400a9cae76(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:17,146 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:17,146 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/C, priority=13, startTime=1733645776660; duration=0sec 2024-12-08T08:16:17,146 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:17,146 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:C 2024-12-08T08:16:17,176 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/2877aa8e11a543e6a41ad996f3d0a45d 2024-12-08T08:16:17,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/aec6ff7d351b4c6cbf4925007d52738b is 50, key is test_row_0/C:col10/1733645776214/Put/seqid=0 2024-12-08T08:16:17,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742298_1474 (size=12001) 2024-12-08T08:16:17,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645837220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:17,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:17,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:17,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:17,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645837241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T08:16:17,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:17,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645837530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:17,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645837561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:17,599 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/aec6ff7d351b4c6cbf4925007d52738b 2024-12-08T08:16:17,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/53e6105f068d405ba3f730995c55432b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/53e6105f068d405ba3f730995c55432b 2024-12-08T08:16:17,617 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/53e6105f068d405ba3f730995c55432b, entries=150, sequenceid=114, filesize=11.7 K 2024-12-08T08:16:17,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/2877aa8e11a543e6a41ad996f3d0a45d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2877aa8e11a543e6a41ad996f3d0a45d 2024-12-08T08:16:17,622 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2877aa8e11a543e6a41ad996f3d0a45d, entries=150, sequenceid=114, filesize=11.7 K 2024-12-08T08:16:17,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/aec6ff7d351b4c6cbf4925007d52738b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/aec6ff7d351b4c6cbf4925007d52738b 2024-12-08T08:16:17,627 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/aec6ff7d351b4c6cbf4925007d52738b, entries=150, sequenceid=114, filesize=11.7 K 2024-12-08T08:16:17,628 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 92943a79353f8896400eee756d87f758 in 914ms, sequenceid=114, compaction requested=false 2024-12-08T08:16:17,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:17,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:17,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-08T08:16:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-08T08:16:17,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-08T08:16:17,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3750 sec 2024-12-08T08:16:17,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.3800 sec 2024-12-08T08:16:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:18,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T08:16:18,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:18,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:18,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:18,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:18,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:18,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:18,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/c3443fcce7d6464d8af54228b51ad77f is 50, key is test_row_0/A:col10/1733645776874/Put/seqid=0 2024-12-08T08:16:18,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742299_1475 (size=14391) 2024-12-08T08:16:18,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/c3443fcce7d6464d8af54228b51ad77f 2024-12-08T08:16:18,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/1a497ef63a414e2fbe9b14e8de313994 is 50, key is test_row_0/B:col10/1733645776874/Put/seqid=0 2024-12-08T08:16:18,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742300_1476 (size=12051) 2024-12-08T08:16:18,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/1a497ef63a414e2fbe9b14e8de313994 2024-12-08T08:16:18,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/394ea432355149fd8cc0872d4aea76fb is 50, key is test_row_0/C:col10/1733645776874/Put/seqid=0 2024-12-08T08:16:18,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742301_1477 (size=12051) 2024-12-08T08:16:18,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645838108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645838110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645838220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645838221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T08:16:18,400 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:18,401 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-08T08:16:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T08:16:18,402 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:18,403 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:18,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:18,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645838430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645838430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/394ea432355149fd8cc0872d4aea76fb 2024-12-08T08:16:18,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/c3443fcce7d6464d8af54228b51ad77f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/c3443fcce7d6464d8af54228b51ad77f 2024-12-08T08:16:18,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/c3443fcce7d6464d8af54228b51ad77f, entries=200, sequenceid=130, filesize=14.1 K 2024-12-08T08:16:18,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/1a497ef63a414e2fbe9b14e8de313994 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/1a497ef63a414e2fbe9b14e8de313994 2024-12-08T08:16:18,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/1a497ef63a414e2fbe9b14e8de313994, entries=150, sequenceid=130, filesize=11.8 K 2024-12-08T08:16:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/394ea432355149fd8cc0872d4aea76fb as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/394ea432355149fd8cc0872d4aea76fb 2024-12-08T08:16:18,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/394ea432355149fd8cc0872d4aea76fb, entries=150, sequenceid=130, filesize=11.8 K 2024-12-08T08:16:18,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 92943a79353f8896400eee756d87f758 in 456ms, sequenceid=130, compaction requested=true 2024-12-08T08:16:18,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:18,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:18,500 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:18,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:18,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:18,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:18,500 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:18,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:18,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:18,501 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:18,501 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:18,501 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/A is initiating minor compaction (all files) 2024-12-08T08:16:18,501 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/B is initiating minor compaction (all files) 2024-12-08T08:16:18,501 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/A in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:18,501 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/B in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:18,501 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2c94800bbf544a53984c9e262c93bf57, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2877aa8e11a543e6a41ad996f3d0a45d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/1a497ef63a414e2fbe9b14e8de313994] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=35.4 K 2024-12-08T08:16:18,501 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/8f235f25bcf24bfaa7ff9b8272ae21a6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/53e6105f068d405ba3f730995c55432b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/c3443fcce7d6464d8af54228b51ad77f] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=37.7 K 2024-12-08T08:16:18,502 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c94800bbf544a53984c9e262c93bf57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733645775451 2024-12-08T08:16:18,502 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f235f25bcf24bfaa7ff9b8272ae21a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733645775451 2024-12-08T08:16:18,502 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53e6105f068d405ba3f730995c55432b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733645776179 2024-12-08T08:16:18,502 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 2877aa8e11a543e6a41ad996f3d0a45d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733645776179 2024-12-08T08:16:18,502 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3443fcce7d6464d8af54228b51ad77f, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733645776874 2024-12-08T08:16:18,502 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a497ef63a414e2fbe9b14e8de313994, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733645776874 2024-12-08T08:16:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:18,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T08:16:18,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:18,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:18,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:18,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:18,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:18,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:18,507 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:18,507 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/2543b98bb5384accbefc24cdac7bdec6 is 50, key is test_row_0/B:col10/1733645776874/Put/seqid=0 2024-12-08T08:16:18,509 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#A#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:18,509 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/ca368e0e17134f4292a8e1449c52881a is 50, key is test_row_0/A:col10/1733645776874/Put/seqid=0 2024-12-08T08:16:18,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/2de9699af3ad45edbdeca08fd6de81ad is 50, key is test_row_0/A:col10/1733645778103/Put/seqid=0 2024-12-08T08:16:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T08:16:18,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742302_1478 (size=12359) 2024-12-08T08:16:18,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742303_1479 (size=12359) 2024-12-08T08:16:18,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742304_1480 (size=14541) 2024-12-08T08:16:18,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/2de9699af3ad45edbdeca08fd6de81ad 2024-12-08T08:16:18,520 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/ca368e0e17134f4292a8e1449c52881a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ca368e0e17134f4292a8e1449c52881a 2024-12-08T08:16:18,524 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/A of 92943a79353f8896400eee756d87f758 into ca368e0e17134f4292a8e1449c52881a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:18,524 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:18,524 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/A, priority=13, startTime=1733645778500; duration=0sec 2024-12-08T08:16:18,524 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:18,524 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:A 2024-12-08T08:16:18,524 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:18,526 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:18,527 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/C is initiating minor compaction (all files) 2024-12-08T08:16:18,527 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/C in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:18,527 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/5d15c66750e74275b1ec9c400a9cae76, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/aec6ff7d351b4c6cbf4925007d52738b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/394ea432355149fd8cc0872d4aea76fb] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=35.4 K 2024-12-08T08:16:18,527 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d15c66750e74275b1ec9c400a9cae76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733645775451 2024-12-08T08:16:18,527 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting aec6ff7d351b4c6cbf4925007d52738b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733645776179 2024-12-08T08:16:18,528 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 394ea432355149fd8cc0872d4aea76fb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733645776874 2024-12-08T08:16:18,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/7bcc8ff18f5c4373afdff365fb8a8cb3 is 50, key is test_row_0/B:col10/1733645778103/Put/seqid=0 2024-12-08T08:16:18,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742305_1481 (size=12151) 2024-12-08T08:16:18,536 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#C#compaction#406 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:18,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645838526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,537 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/819ff20ca7344ccebfb6245014ebb259 is 50, key is test_row_0/C:col10/1733645776874/Put/seqid=0 2024-12-08T08:16:18,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645838527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645838529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742306_1482 (size=12359) 2024-12-08T08:16:18,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T08:16:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:18,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:18,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645838641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645838641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645838641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T08:16:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:18,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T08:16:18,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645838740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645838740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T08:16:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645838850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:18,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:18,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:18,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645838850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645838850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:18,917 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/2543b98bb5384accbefc24cdac7bdec6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2543b98bb5384accbefc24cdac7bdec6 2024-12-08T08:16:18,921 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/B of 92943a79353f8896400eee756d87f758 into 2543b98bb5384accbefc24cdac7bdec6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:18,922 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:18,922 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/B, priority=13, startTime=1733645778500; duration=0sec 2024-12-08T08:16:18,922 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:18,922 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:B 2024-12-08T08:16:18,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/7bcc8ff18f5c4373afdff365fb8a8cb3 2024-12-08T08:16:18,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/bc8c1b596dc946b298d861f653a8502c is 50, key is test_row_0/C:col10/1733645778103/Put/seqid=0 2024-12-08T08:16:18,951 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/819ff20ca7344ccebfb6245014ebb259 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/819ff20ca7344ccebfb6245014ebb259 2024-12-08T08:16:18,963 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/C of 92943a79353f8896400eee756d87f758 into 819ff20ca7344ccebfb6245014ebb259(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:18,964 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:18,964 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/C, priority=13, startTime=1733645778500; duration=0sec 2024-12-08T08:16:18,964 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:18,964 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:C 2024-12-08T08:16:18,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742307_1483 (size=12151) 2024-12-08T08:16:18,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/bc8c1b596dc946b298d861f653a8502c 2024-12-08T08:16:18,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/2de9699af3ad45edbdeca08fd6de81ad as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/2de9699af3ad45edbdeca08fd6de81ad 2024-12-08T08:16:18,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/2de9699af3ad45edbdeca08fd6de81ad, entries=200, sequenceid=150, filesize=14.2 K 2024-12-08T08:16:18,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/7bcc8ff18f5c4373afdff365fb8a8cb3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/7bcc8ff18f5c4373afdff365fb8a8cb3 2024-12-08T08:16:18,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/7bcc8ff18f5c4373afdff365fb8a8cb3, entries=150, sequenceid=150, filesize=11.9 K 2024-12-08T08:16:18,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/bc8c1b596dc946b298d861f653a8502c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/bc8c1b596dc946b298d861f653a8502c 2024-12-08T08:16:19,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/bc8c1b596dc946b298d861f653a8502c, entries=150, sequenceid=150, filesize=11.9 K 2024-12-08T08:16:19,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 92943a79353f8896400eee756d87f758 in 498ms, sequenceid=150, compaction requested=false 2024-12-08T08:16:19,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:19,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T08:16:19,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:19,013 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T08:16:19,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:19,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:19,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:19,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:19,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:19,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:19,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/e520a73e8b3347bca5b91b55aba73dfa is 50, key is test_row_0/A:col10/1733645778526/Put/seqid=0 2024-12-08T08:16:19,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742308_1484 (size=12151) 2024-12-08T08:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T08:16:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:19,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:19,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645839220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645839220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:19,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645839221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:19,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645839250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645839251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:19,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645839330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645839330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:19,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645839341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:19,420 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/e520a73e8b3347bca5b91b55aba73dfa 2024-12-08T08:16:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/9f14665d52de4cbfacd5571a24f65129 is 50, key is test_row_0/B:col10/1733645778526/Put/seqid=0 2024-12-08T08:16:19,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742309_1485 (size=12151) 2024-12-08T08:16:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T08:16:19,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645839550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645839550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645839551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:19,830 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/9f14665d52de4cbfacd5571a24f65129 2024-12-08T08:16:19,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/1681f1506c7544f6a3016342ed2ee585 is 50, key is test_row_0/C:col10/1733645778526/Put/seqid=0 2024-12-08T08:16:19,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645839861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:19,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742310_1486 (size=12151) 2024-12-08T08:16:19,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645839861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:19,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645839871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:20,272 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/1681f1506c7544f6a3016342ed2ee585 2024-12-08T08:16:20,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/e520a73e8b3347bca5b91b55aba73dfa as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/e520a73e8b3347bca5b91b55aba73dfa 2024-12-08T08:16:20,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645840270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:20,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645840270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:20,277 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 92943a79353f8896400eee756d87f758 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:16:20,277 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_0' in 92943a79353f8896400eee756d87f758 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:16:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:20,278 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/e520a73e8b3347bca5b91b55aba73dfa, entries=150, sequenceid=165, filesize=11.9 K 2024-12-08T08:16:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/9f14665d52de4cbfacd5571a24f65129 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9f14665d52de4cbfacd5571a24f65129 2024-12-08T08:16:20,281 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9f14665d52de4cbfacd5571a24f65129, entries=150, sequenceid=165, filesize=11.9 K 2024-12-08T08:16:20,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/1681f1506c7544f6a3016342ed2ee585 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/1681f1506c7544f6a3016342ed2ee585 2024-12-08T08:16:20,285 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/1681f1506c7544f6a3016342ed2ee585, entries=150, sequenceid=165, filesize=11.9 K 2024-12-08T08:16:20,286 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 92943a79353f8896400eee756d87f758 in 1273ms, sequenceid=165, compaction requested=true 2024-12-08T08:16:20,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:20,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:20,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-08T08:16:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-08T08:16:20,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-08T08:16:20,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8840 sec 2024-12-08T08:16:20,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.8880 sec 2024-12-08T08:16:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:20,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-08T08:16:20,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:20,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/79827dc05ee548d69560ebe50c33d765 is 50, key is test_row_0/A:col10/1733645779219/Put/seqid=0 2024-12-08T08:16:20,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742311_1487 (size=14541) 2024-12-08T08:16:20,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645840399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:20,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645840407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:20,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645840407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:20,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645840511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:20,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645840520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:20,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645840520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T08:16:20,551 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:20,552 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-08T08:16:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T08:16:20,556 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:20,557 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:20,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T08:16:20,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T08:16:20,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:20,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:20,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:20,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:20,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:20,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645840720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:20,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645840730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:20,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645840730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:20,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/79827dc05ee548d69560ebe50c33d765 2024-12-08T08:16:20,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/7bc30169500e402d8a56388092598e87 is 50, key is test_row_0/B:col10/1733645779219/Put/seqid=0 2024-12-08T08:16:20,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742312_1488 (size=12151) 2024-12-08T08:16:20,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T08:16:20,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:20,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:20,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:20,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:20,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T08:16:21,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T08:16:21,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:21,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:21,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:21,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:21,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:21,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645841041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:21,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645841050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:21,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645841050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:21,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T08:16:21,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:21,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:21,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:21,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:21,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T08:16:21,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/7bc30169500e402d8a56388092598e87 2024-12-08T08:16:21,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/bdf2c78cc1fb44e1ad5ad310930f575d is 50, key is test_row_0/C:col10/1733645779219/Put/seqid=0 2024-12-08T08:16:21,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742313_1489 (size=12151) 2024-12-08T08:16:21,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/bdf2c78cc1fb44e1ad5ad310930f575d 2024-12-08T08:16:21,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/79827dc05ee548d69560ebe50c33d765 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/79827dc05ee548d69560ebe50c33d765 2024-12-08T08:16:21,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/79827dc05ee548d69560ebe50c33d765, entries=200, sequenceid=186, filesize=14.2 K 2024-12-08T08:16:21,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/7bc30169500e402d8a56388092598e87 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/7bc30169500e402d8a56388092598e87 2024-12-08T08:16:21,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/7bc30169500e402d8a56388092598e87, entries=150, sequenceid=186, filesize=11.9 K 2024-12-08T08:16:21,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/bdf2c78cc1fb44e1ad5ad310930f575d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/bdf2c78cc1fb44e1ad5ad310930f575d 2024-12-08T08:16:21,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/bdf2c78cc1fb44e1ad5ad310930f575d, entries=150, sequenceid=186, filesize=11.9 K 2024-12-08T08:16:21,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 92943a79353f8896400eee756d87f758 in 866ms, sequenceid=186, compaction requested=true 2024-12-08T08:16:21,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:21,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:21,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:21,252 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:16:21,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:21,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:21,252 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:16:21,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:21,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:21,253 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:16:21,253 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:16:21,253 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/A is initiating minor compaction (all files) 2024-12-08T08:16:21,253 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/B is initiating minor compaction (all files) 2024-12-08T08:16:21,253 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/B in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:21,253 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/A in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:21,253 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2543b98bb5384accbefc24cdac7bdec6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/7bcc8ff18f5c4373afdff365fb8a8cb3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9f14665d52de4cbfacd5571a24f65129, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/7bc30169500e402d8a56388092598e87] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=47.7 K 2024-12-08T08:16:21,253 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ca368e0e17134f4292a8e1449c52881a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/2de9699af3ad45edbdeca08fd6de81ad, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/e520a73e8b3347bca5b91b55aba73dfa, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/79827dc05ee548d69560ebe50c33d765] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=52.3 K 2024-12-08T08:16:21,254 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 2543b98bb5384accbefc24cdac7bdec6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733645776874 2024-12-08T08:16:21,254 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca368e0e17134f4292a8e1449c52881a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733645776874 2024-12-08T08:16:21,254 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bcc8ff18f5c4373afdff365fb8a8cb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733645778103 2024-12-08T08:16:21,254 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2de9699af3ad45edbdeca08fd6de81ad, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733645778103 2024-12-08T08:16:21,254 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f14665d52de4cbfacd5571a24f65129, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733645778525 2024-12-08T08:16:21,254 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e520a73e8b3347bca5b91b55aba73dfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733645778525 2024-12-08T08:16:21,254 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bc30169500e402d8a56388092598e87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733645779218 2024-12-08T08:16:21,254 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79827dc05ee548d69560ebe50c33d765, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733645779205 2024-12-08T08:16:21,261 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:21,262 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/bbebf197e0b14813afdd956d22d74edc is 50, key is test_row_0/B:col10/1733645779219/Put/seqid=0 2024-12-08T08:16:21,262 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:21,262 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/7e56afbcd0a34afca0d8f077e9957acf is 50, key is test_row_0/A:col10/1733645779219/Put/seqid=0 2024-12-08T08:16:21,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742314_1490 (size=12595) 2024-12-08T08:16:21,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742315_1491 (size=12595) 2024-12-08T08:16:21,277 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/7e56afbcd0a34afca0d8f077e9957acf as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/7e56afbcd0a34afca0d8f077e9957acf 2024-12-08T08:16:21,283 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 92943a79353f8896400eee756d87f758/A of 92943a79353f8896400eee756d87f758 into 7e56afbcd0a34afca0d8f077e9957acf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:21,283 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:21,283 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/A, priority=12, startTime=1733645781252; duration=0sec 2024-12-08T08:16:21,283 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:21,283 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:A 2024-12-08T08:16:21,283 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:16:21,284 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:16:21,284 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/C is initiating minor compaction (all files) 2024-12-08T08:16:21,284 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/C in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:21,284 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/819ff20ca7344ccebfb6245014ebb259, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/bc8c1b596dc946b298d861f653a8502c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/1681f1506c7544f6a3016342ed2ee585, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/bdf2c78cc1fb44e1ad5ad310930f575d] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=47.7 K 2024-12-08T08:16:21,285 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 819ff20ca7344ccebfb6245014ebb259, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733645776874 2024-12-08T08:16:21,285 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc8c1b596dc946b298d861f653a8502c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733645778103 2024-12-08T08:16:21,285 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1681f1506c7544f6a3016342ed2ee585, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733645778525 2024-12-08T08:16:21,285 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdf2c78cc1fb44e1ad5ad310930f575d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733645779218 2024-12-08T08:16:21,294 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:21,295 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/4fe99dcb442f41c39f6941f2437c4cff is 50, key is test_row_0/C:col10/1733645779219/Put/seqid=0 2024-12-08T08:16:21,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742316_1492 (size=12595) 2024-12-08T08:16:21,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T08:16:21,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:21,321 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T08:16:21,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:21,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:21,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:21,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:21,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:21,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:21,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/ac66f282c6224eea8bc8c0821a2c6157 is 50, key is test_row_0/A:col10/1733645780406/Put/seqid=0 2024-12-08T08:16:21,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742317_1493 (size=12151) 2024-12-08T08:16:21,350 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/ac66f282c6224eea8bc8c0821a2c6157 2024-12-08T08:16:21,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/9a713ecf06f04c85bf30adaee92f2855 is 50, key is test_row_0/B:col10/1733645780406/Put/seqid=0 2024-12-08T08:16:21,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742318_1494 (size=12151) 2024-12-08T08:16:21,375 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/9a713ecf06f04c85bf30adaee92f2855 2024-12-08T08:16:21,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/2be0c255057545499d6195149f94b739 is 50, key is test_row_0/C:col10/1733645780406/Put/seqid=0 2024-12-08T08:16:21,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742319_1495 (size=12151) 2024-12-08T08:16:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:21,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:21,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645841632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:21,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645841632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:21,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645781614, max timestamp=1733645781644Num uniques 859; 2024-12-08T08:16:21,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645781614, max timestamp=1733645781644Num uniques 859; 2024-12-08T08:16:21,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645781614, max timestamp=1733645781644Num uniques 859; 2024-12-08T08:16:21,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:16:21,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:16:21,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:16:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:21,674 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/bbebf197e0b14813afdd956d22d74edc as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/bbebf197e0b14813afdd956d22d74edc 2024-12-08T08:16:21,682 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 92943a79353f8896400eee756d87f758/B of 92943a79353f8896400eee756d87f758 into bbebf197e0b14813afdd956d22d74edc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:21,682 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:21,682 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/B, priority=12, startTime=1733645781252; duration=0sec 2024-12-08T08:16:21,682 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:21,682 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:B 2024-12-08T08:16:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T08:16:21,710 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/4fe99dcb442f41c39f6941f2437c4cff as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/4fe99dcb442f41c39f6941f2437c4cff 2024-12-08T08:16:21,714 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 92943a79353f8896400eee756d87f758/C of 92943a79353f8896400eee756d87f758 into 4fe99dcb442f41c39f6941f2437c4cff(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:21,714 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:21,714 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/C, priority=12, startTime=1733645781252; duration=0sec 2024-12-08T08:16:21,714 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:21,715 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:C 2024-12-08T08:16:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645841769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645841773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:21,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645841777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:21,840 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/2be0c255057545499d6195149f94b739 2024-12-08T08:16:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/ac66f282c6224eea8bc8c0821a2c6157 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ac66f282c6224eea8bc8c0821a2c6157 2024-12-08T08:16:21,847 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ac66f282c6224eea8bc8c0821a2c6157, entries=150, sequenceid=198, filesize=11.9 K 2024-12-08T08:16:21,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/9a713ecf06f04c85bf30adaee92f2855 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9a713ecf06f04c85bf30adaee92f2855 2024-12-08T08:16:21,850 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9a713ecf06f04c85bf30adaee92f2855, entries=150, sequenceid=198, filesize=11.9 K 2024-12-08T08:16:21,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/2be0c255057545499d6195149f94b739 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/2be0c255057545499d6195149f94b739 2024-12-08T08:16:21,858 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/2be0c255057545499d6195149f94b739, entries=150, sequenceid=198, filesize=11.9 K 2024-12-08T08:16:21,859 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 92943a79353f8896400eee756d87f758 in 538ms, sequenceid=198, compaction requested=false 2024-12-08T08:16:21,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:21,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:21,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-08T08:16:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-08T08:16:21,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-08T08:16:21,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3030 sec 2024-12-08T08:16:21,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.3110 sec 2024-12-08T08:16:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:21,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-08T08:16:21,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:21,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:21,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:21,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:21,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:21,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:21,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/1faf1d1f85004b5f8142299a1ad4f9be is 50, key is test_row_0/A:col10/1733645781746/Put/seqid=0 2024-12-08T08:16:21,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742320_1496 (size=14541) 2024-12-08T08:16:21,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/1faf1d1f85004b5f8142299a1ad4f9be 2024-12-08T08:16:21,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/4be58442490c4aed89161e7046aff3af is 50, key is test_row_0/B:col10/1733645781746/Put/seqid=0 2024-12-08T08:16:21,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742321_1497 (size=12151) 2024-12-08T08:16:21,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/4be58442490c4aed89161e7046aff3af 2024-12-08T08:16:21,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645841920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:21,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645841921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645841923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:21,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/e5b5bc5e911b4ed78201eb214d6a78be is 50, key is test_row_0/C:col10/1733645781746/Put/seqid=0 2024-12-08T08:16:21,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742322_1498 (size=12151) 2024-12-08T08:16:21,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/e5b5bc5e911b4ed78201eb214d6a78be 2024-12-08T08:16:21,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/1faf1d1f85004b5f8142299a1ad4f9be as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/1faf1d1f85004b5f8142299a1ad4f9be 2024-12-08T08:16:21,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/1faf1d1f85004b5f8142299a1ad4f9be, entries=200, sequenceid=237, filesize=14.2 K 2024-12-08T08:16:21,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/4be58442490c4aed89161e7046aff3af as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/4be58442490c4aed89161e7046aff3af 2024-12-08T08:16:21,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/4be58442490c4aed89161e7046aff3af, entries=150, sequenceid=237, filesize=11.9 K 2024-12-08T08:16:21,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/e5b5bc5e911b4ed78201eb214d6a78be as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e5b5bc5e911b4ed78201eb214d6a78be 2024-12-08T08:16:21,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e5b5bc5e911b4ed78201eb214d6a78be, entries=150, sequenceid=237, filesize=11.9 K 2024-12-08T08:16:21,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 92943a79353f8896400eee756d87f758 in 83ms, sequenceid=237, compaction requested=true 2024-12-08T08:16:21,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:21,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:21,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:21,979 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:21,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:21,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:21,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:21,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T08:16:21,979 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:21,979 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:21,979 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/A is initiating minor compaction (all files) 2024-12-08T08:16:21,980 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/A in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:21,980 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:21,980 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/7e56afbcd0a34afca0d8f077e9957acf, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ac66f282c6224eea8bc8c0821a2c6157, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/1faf1d1f85004b5f8142299a1ad4f9be] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=38.4 K 2024-12-08T08:16:21,980 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/B is initiating minor compaction (all files) 2024-12-08T08:16:21,980 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/B in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:21,980 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/bbebf197e0b14813afdd956d22d74edc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9a713ecf06f04c85bf30adaee92f2855, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/4be58442490c4aed89161e7046aff3af] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=36.0 K 2024-12-08T08:16:21,981 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting bbebf197e0b14813afdd956d22d74edc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733645779218 2024-12-08T08:16:21,981 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e56afbcd0a34afca0d8f077e9957acf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733645779218 2024-12-08T08:16:21,981 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a713ecf06f04c85bf30adaee92f2855, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733645780396 2024-12-08T08:16:21,981 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac66f282c6224eea8bc8c0821a2c6157, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733645780396 2024-12-08T08:16:21,981 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 4be58442490c4aed89161e7046aff3af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733645781746 2024-12-08T08:16:21,982 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1faf1d1f85004b5f8142299a1ad4f9be, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733645781746 2024-12-08T08:16:22,001 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:22,001 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/e9e9493f5d26498c84f49de8506feea3 is 50, key is test_row_0/B:col10/1733645781746/Put/seqid=0 2024-12-08T08:16:22,009 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#A#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:22,010 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/3d6a1adba8ec4e0883851225ffc03559 is 50, key is test_row_0/A:col10/1733645781746/Put/seqid=0 2024-12-08T08:16:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:22,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T08:16:22,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:22,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:22,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:22,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:22,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:22,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:22,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742324_1500 (size=12697) 2024-12-08T08:16:22,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742323_1499 (size=12697) 2024-12-08T08:16:22,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/cf524a96902449d686595291c86b1250 is 50, key is test_row_0/A:col10/1733645782044/Put/seqid=0 2024-12-08T08:16:22,064 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/3d6a1adba8ec4e0883851225ffc03559 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/3d6a1adba8ec4e0883851225ffc03559 2024-12-08T08:16:22,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742325_1501 (size=14541) 2024-12-08T08:16:22,069 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/e9e9493f5d26498c84f49de8506feea3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/e9e9493f5d26498c84f49de8506feea3 2024-12-08T08:16:22,070 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/A of 92943a79353f8896400eee756d87f758 into 3d6a1adba8ec4e0883851225ffc03559(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:22,070 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:22,070 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/A, priority=13, startTime=1733645781978; duration=0sec 2024-12-08T08:16:22,070 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:22,070 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:A 2024-12-08T08:16:22,070 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:22,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/cf524a96902449d686595291c86b1250 2024-12-08T08:16:22,073 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:22,073 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/C is initiating minor compaction (all files) 2024-12-08T08:16:22,073 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/C in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:22,073 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/4fe99dcb442f41c39f6941f2437c4cff, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/2be0c255057545499d6195149f94b739, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e5b5bc5e911b4ed78201eb214d6a78be] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=36.0 K 2024-12-08T08:16:22,073 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fe99dcb442f41c39f6941f2437c4cff, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733645779218 2024-12-08T08:16:22,074 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2be0c255057545499d6195149f94b739, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733645780396 2024-12-08T08:16:22,075 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5b5bc5e911b4ed78201eb214d6a78be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733645781746 2024-12-08T08:16:22,079 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/B of 92943a79353f8896400eee756d87f758 into e9e9493f5d26498c84f49de8506feea3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:22,079 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:22,079 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/B, priority=13, startTime=1733645781979; duration=0sec 2024-12-08T08:16:22,080 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:22,080 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:B 2024-12-08T08:16:22,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/0f6bca2de98a416f8ad297de285e15b5 is 50, key is test_row_0/B:col10/1733645782044/Put/seqid=0 2024-12-08T08:16:22,091 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#C#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:22,091 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/4bada5794fa7426d9d8fb503fee748f7 is 50, key is test_row_0/C:col10/1733645781746/Put/seqid=0 2024-12-08T08:16:22,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742326_1502 (size=12697) 2024-12-08T08:16:22,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733645782072, max timestamp=1733645782116Num uniques 859; 2024-12-08T08:16:22,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:16:22,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733645782072, max timestamp=1733645782116Num uniques 859; 2024-12-08T08:16:22,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:16:22,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733645782072, max timestamp=1733645782116Num uniques 859; 2024-12-08T08:16:22,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:16:22,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742327_1503 (size=12151) 2024-12-08T08:16:22,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645842181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645842181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645842181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645842301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645842301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645842301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645842301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645842301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,315 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_0' in 92943a79353f8896400eee756d87f758 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:16:22,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,316 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 92943a79353f8896400eee756d87f758 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:16:22,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,509 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/4bada5794fa7426d9d8fb503fee748f7 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/4bada5794fa7426d9d8fb503fee748f7 2024-12-08T08:16:22,513 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/C of 92943a79353f8896400eee756d87f758 into 4bada5794fa7426d9d8fb503fee748f7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:22,513 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:22,513 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/C, priority=13, startTime=1733645781979; duration=0sec 2024-12-08T08:16:22,513 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:22,513 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:C 2024-12-08T08:16:22,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/0f6bca2de98a416f8ad297de285e15b5 2024-12-08T08:16:22,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645842521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645842521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645842521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/c7389266c62a4f4da80623fc8c33d972 is 50, key is test_row_0/C:col10/1733645782044/Put/seqid=0 2024-12-08T08:16:22,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742328_1504 (size=12151) 2024-12-08T08:16:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T08:16:22,711 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:22,712 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-08T08:16:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T08:16:22,716 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:22,717 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:22,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T08:16:22,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645842831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645842831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645842831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:22,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:22,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:22,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T08:16:22,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:22,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:22,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:22,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:22,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:22,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/c7389266c62a4f4da80623fc8c33d972 2024-12-08T08:16:22,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/cf524a96902449d686595291c86b1250 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cf524a96902449d686595291c86b1250 2024-12-08T08:16:22,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cf524a96902449d686595291c86b1250, entries=200, sequenceid=248, filesize=14.2 K 2024-12-08T08:16:22,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/0f6bca2de98a416f8ad297de285e15b5 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/0f6bca2de98a416f8ad297de285e15b5 2024-12-08T08:16:22,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/0f6bca2de98a416f8ad297de285e15b5, entries=150, sequenceid=248, filesize=11.9 K 2024-12-08T08:16:22,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/c7389266c62a4f4da80623fc8c33d972 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/c7389266c62a4f4da80623fc8c33d972 2024-12-08T08:16:22,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/c7389266c62a4f4da80623fc8c33d972, entries=150, sequenceid=248, filesize=11.9 K 2024-12-08T08:16:22,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 92943a79353f8896400eee756d87f758 in 917ms, sequenceid=248, compaction requested=false 2024-12-08T08:16:22,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:23,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T08:16:23,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:23,026 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-08T08:16:23,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:23,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:23,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:23,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:23,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:23,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T08:16:23,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/8685735bb53b47fc9b3ac68c1d60a1c7 is 50, key is test_row_0/A:col10/1733645782176/Put/seqid=0 2024-12-08T08:16:23,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742329_1505 (size=12301) 2024-12-08T08:16:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T08:16:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:23,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:23,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645843365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:23,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645843366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:23,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645843371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:23,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:23,446 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/8685735bb53b47fc9b3ac68c1d60a1c7 2024-12-08T08:16:23,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/9ee9e684b78f4e109778b17010503d52 is 50, key is test_row_0/B:col10/1733645782176/Put/seqid=0 2024-12-08T08:16:23,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742330_1506 (size=12301) 2024-12-08T08:16:23,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645843480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:23,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:23,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645843480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:23,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645843491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:23,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645843690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:23,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645843690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:23,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645843710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T08:16:23,865 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/9ee9e684b78f4e109778b17010503d52 2024-12-08T08:16:23,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/55ef200cc35443b19bf2fcd9188b35a9 is 50, key is test_row_0/C:col10/1733645782176/Put/seqid=0 2024-12-08T08:16:23,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742331_1507 (size=12301) 2024-12-08T08:16:23,876 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/55ef200cc35443b19bf2fcd9188b35a9 2024-12-08T08:16:23,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/8685735bb53b47fc9b3ac68c1d60a1c7 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/8685735bb53b47fc9b3ac68c1d60a1c7 2024-12-08T08:16:23,881 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/8685735bb53b47fc9b3ac68c1d60a1c7, entries=150, sequenceid=285, filesize=12.0 K 2024-12-08T08:16:23,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/9ee9e684b78f4e109778b17010503d52 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9ee9e684b78f4e109778b17010503d52 2024-12-08T08:16:23,885 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9ee9e684b78f4e109778b17010503d52, entries=150, sequenceid=285, filesize=12.0 K 2024-12-08T08:16:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/55ef200cc35443b19bf2fcd9188b35a9 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/55ef200cc35443b19bf2fcd9188b35a9 2024-12-08T08:16:23,888 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/55ef200cc35443b19bf2fcd9188b35a9, entries=150, sequenceid=285, filesize=12.0 K 2024-12-08T08:16:23,888 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 92943a79353f8896400eee756d87f758 in 862ms, sequenceid=285, compaction requested=true 2024-12-08T08:16:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-08T08:16:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-08T08:16:23,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-08T08:16:23,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1730 sec 2024-12-08T08:16:23,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.1790 sec 2024-12-08T08:16:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:24,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T08:16:24,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:24,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:24,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:24,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:24,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:24,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:24,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/089c993fde384cac9c4e7e4789b75de0 is 50, key is test_row_0/A:col10/1733645783365/Put/seqid=0 2024-12-08T08:16:24,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742332_1508 (size=14741) 2024-12-08T08:16:24,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645844037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:24,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645844043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:24,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645844044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:24,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645844150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:24,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645844150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:24,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645844160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:24,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645844360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:24,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645844360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:24,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645844370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:24,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/089c993fde384cac9c4e7e4789b75de0 2024-12-08T08:16:24,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/ded17e2cc63a4e61aa1dbaebce6a0936 is 50, key is test_row_0/B:col10/1733645783365/Put/seqid=0 2024-12-08T08:16:24,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742333_1509 (size=12301) 2024-12-08T08:16:24,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645844671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:24,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645844671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:24,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645844681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:24,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/ded17e2cc63a4e61aa1dbaebce6a0936 2024-12-08T08:16:24,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/3168a7f0fa164f20a1ee10daf310900c is 50, key is test_row_0/C:col10/1733645783365/Put/seqid=0 2024-12-08T08:16:24,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742334_1510 (size=12301) 2024-12-08T08:16:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T08:16:24,860 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:24,861 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-08T08:16:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T08:16:24,862 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:24,863 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:24,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T08:16:25,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T08:16:25,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:25,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:25,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:25,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:25,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:25,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T08:16:25,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:25,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:25,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:25,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:25,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T08:16:25,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645845181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:25,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645845181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:25,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645845190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:25,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/3168a7f0fa164f20a1ee10daf310900c 2024-12-08T08:16:25,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/089c993fde384cac9c4e7e4789b75de0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/089c993fde384cac9c4e7e4789b75de0 2024-12-08T08:16:25,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/089c993fde384cac9c4e7e4789b75de0, entries=200, sequenceid=297, filesize=14.4 K 2024-12-08T08:16:25,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/ded17e2cc63a4e61aa1dbaebce6a0936 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/ded17e2cc63a4e61aa1dbaebce6a0936 2024-12-08T08:16:25,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/ded17e2cc63a4e61aa1dbaebce6a0936, entries=150, sequenceid=297, filesize=12.0 K 2024-12-08T08:16:25,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/3168a7f0fa164f20a1ee10daf310900c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/3168a7f0fa164f20a1ee10daf310900c 2024-12-08T08:16:25,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/3168a7f0fa164f20a1ee10daf310900c, entries=150, sequenceid=297, filesize=12.0 K 2024-12-08T08:16:25,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 92943a79353f8896400eee756d87f758 in 1237ms, sequenceid=297, compaction requested=true 2024-12-08T08:16:25,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:25,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:25,241 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:16:25,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:25,241 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:16:25,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:25,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:25,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:25,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:25,242 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54280 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:16:25,242 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:16:25,242 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/A is initiating minor compaction (all files) 2024-12-08T08:16:25,242 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/B is initiating minor compaction (all files) 2024-12-08T08:16:25,242 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/B in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:25,242 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/A in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:25,242 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/3d6a1adba8ec4e0883851225ffc03559, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cf524a96902449d686595291c86b1250, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/8685735bb53b47fc9b3ac68c1d60a1c7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/089c993fde384cac9c4e7e4789b75de0] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=53.0 K 2024-12-08T08:16:25,242 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/e9e9493f5d26498c84f49de8506feea3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/0f6bca2de98a416f8ad297de285e15b5, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9ee9e684b78f4e109778b17010503d52, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/ded17e2cc63a4e61aa1dbaebce6a0936] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=48.3 K 2024-12-08T08:16:25,242 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d6a1adba8ec4e0883851225ffc03559, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733645781746 2024-12-08T08:16:25,242 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting e9e9493f5d26498c84f49de8506feea3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733645781746 2024-12-08T08:16:25,242 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f6bca2de98a416f8ad297de285e15b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733645781916 2024-12-08T08:16:25,242 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf524a96902449d686595291c86b1250, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733645781916 2024-12-08T08:16:25,243 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ee9e684b78f4e109778b17010503d52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733645782162 2024-12-08T08:16:25,243 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8685735bb53b47fc9b3ac68c1d60a1c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733645782162 2024-12-08T08:16:25,243 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting ded17e2cc63a4e61aa1dbaebce6a0936, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733645783364 2024-12-08T08:16:25,243 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 089c993fde384cac9c4e7e4789b75de0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733645783364 2024-12-08T08:16:25,249 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:25,249 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/98394bd14c744dbcb09a4401d52520b4 is 50, key is test_row_0/B:col10/1733645783365/Put/seqid=0 2024-12-08T08:16:25,250 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#A#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:25,250 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/49bb4a687afb474dad4bda6eabb11406 is 50, key is test_row_0/A:col10/1733645783365/Put/seqid=0 2024-12-08T08:16:25,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742335_1511 (size=12983) 2024-12-08T08:16:25,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742336_1512 (size=12983) 2024-12-08T08:16:25,274 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/49bb4a687afb474dad4bda6eabb11406 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/49bb4a687afb474dad4bda6eabb11406 2024-12-08T08:16:25,277 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 92943a79353f8896400eee756d87f758/A of 92943a79353f8896400eee756d87f758 into 49bb4a687afb474dad4bda6eabb11406(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:25,277 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:25,277 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/A, priority=12, startTime=1733645785241; duration=0sec 2024-12-08T08:16:25,277 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:25,277 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:A 2024-12-08T08:16:25,277 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:16:25,278 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:16:25,278 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/C is initiating minor compaction (all files) 2024-12-08T08:16:25,278 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/C in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:25,278 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/4bada5794fa7426d9d8fb503fee748f7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/c7389266c62a4f4da80623fc8c33d972, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/55ef200cc35443b19bf2fcd9188b35a9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/3168a7f0fa164f20a1ee10daf310900c] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=48.3 K 2024-12-08T08:16:25,279 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bada5794fa7426d9d8fb503fee748f7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733645781746 2024-12-08T08:16:25,279 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7389266c62a4f4da80623fc8c33d972, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733645781916 2024-12-08T08:16:25,279 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55ef200cc35443b19bf2fcd9188b35a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733645782162 2024-12-08T08:16:25,279 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3168a7f0fa164f20a1ee10daf310900c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733645783364 2024-12-08T08:16:25,288 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#C#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:25,289 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/36123c96cea04fdbb785048d49d80862 is 50, key is test_row_0/C:col10/1733645783365/Put/seqid=0 2024-12-08T08:16:25,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742337_1513 (size=12983) 2024-12-08T08:16:25,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T08:16:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:25,320 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T08:16:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:25,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:25,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:25,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/d55f9dff3b174d8cb6b8a4c4262af0ad is 50, key is test_row_0/A:col10/1733645784041/Put/seqid=0 2024-12-08T08:16:25,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742338_1514 (size=12301) 2024-12-08T08:16:25,331 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/d55f9dff3b174d8cb6b8a4c4262af0ad 2024-12-08T08:16:25,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/53dd035c000948f99a2396b70209fe5d is 50, key is test_row_0/B:col10/1733645784041/Put/seqid=0 2024-12-08T08:16:25,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742339_1515 (size=12301) 2024-12-08T08:16:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T08:16:25,663 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/98394bd14c744dbcb09a4401d52520b4 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/98394bd14c744dbcb09a4401d52520b4 2024-12-08T08:16:25,667 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 92943a79353f8896400eee756d87f758/B of 92943a79353f8896400eee756d87f758 into 98394bd14c744dbcb09a4401d52520b4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:25,667 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:25,667 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/B, priority=12, startTime=1733645785241; duration=0sec 2024-12-08T08:16:25,667 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:25,667 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:B 2024-12-08T08:16:25,707 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/36123c96cea04fdbb785048d49d80862 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/36123c96cea04fdbb785048d49d80862 2024-12-08T08:16:25,712 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 92943a79353f8896400eee756d87f758/C of 92943a79353f8896400eee756d87f758 into 36123c96cea04fdbb785048d49d80862(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:25,712 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:25,712 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/C, priority=12, startTime=1733645785241; duration=0sec 2024-12-08T08:16:25,712 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:25,712 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:C 2024-12-08T08:16:25,749 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/53dd035c000948f99a2396b70209fe5d 2024-12-08T08:16:25,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/10c2c628235b4194b366143d25e37263 is 50, key is test_row_0/C:col10/1733645784041/Put/seqid=0 2024-12-08T08:16:25,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742340_1516 (size=12301) 2024-12-08T08:16:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T08:16:26,157 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/10c2c628235b4194b366143d25e37263 2024-12-08T08:16:26,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/d55f9dff3b174d8cb6b8a4c4262af0ad as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/d55f9dff3b174d8cb6b8a4c4262af0ad 2024-12-08T08:16:26,173 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/d55f9dff3b174d8cb6b8a4c4262af0ad, entries=150, sequenceid=318, filesize=12.0 K 2024-12-08T08:16:26,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/53dd035c000948f99a2396b70209fe5d as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/53dd035c000948f99a2396b70209fe5d 2024-12-08T08:16:26,176 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/53dd035c000948f99a2396b70209fe5d, entries=150, sequenceid=318, filesize=12.0 K 2024-12-08T08:16:26,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/10c2c628235b4194b366143d25e37263 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/10c2c628235b4194b366143d25e37263 2024-12-08T08:16:26,180 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/10c2c628235b4194b366143d25e37263, entries=150, sequenceid=318, filesize=12.0 K 2024-12-08T08:16:26,181 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 92943a79353f8896400eee756d87f758 in 861ms, sequenceid=318, compaction requested=false 2024-12-08T08:16:26,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:26,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:26,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-08T08:16:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-08T08:16:26,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-08T08:16:26,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3190 sec 2024-12-08T08:16:26,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.3230 sec 2024-12-08T08:16:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:26,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:16:26,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:26,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:26,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:26,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:26,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:26,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:26,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/cd1271604c054968ac1996580b3822b4 is 50, key is test_row_0/A:col10/1733645786203/Put/seqid=0 2024-12-08T08:16:26,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742341_1517 (size=17181) 2024-12-08T08:16:26,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733645786261, max timestamp=1733645786269Num uniques 859; 2024-12-08T08:16:26,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733645786261, max timestamp=1733645786269Num uniques 859; 2024-12-08T08:16:26,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733645786261, max timestamp=1733645786269Num uniques 859; 2024-12-08T08:16:26,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:16:26,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:16:26,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:16:26,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645846303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:26,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645846304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645846305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:26,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38464 deadline: 1733645846330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38448 deadline: 1733645846330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,335 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_0' in 92943a79353f8896400eee756d87f758 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:16:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:26,335 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 92943a79353f8896400eee756d87f758 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:16:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:26,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645846411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:26,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645846420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645846420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:26,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645846621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:26,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645846631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:26,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645846631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:26,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/cd1271604c054968ac1996580b3822b4 2024-12-08T08:16:26,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/005c5cc11d3a4da2afef57717d8635dd is 50, key is test_row_0/B:col10/1733645786203/Put/seqid=0 2024-12-08T08:16:26,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742342_1518 (size=12301) 2024-12-08T08:16:26,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645846940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645846940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:26,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645846950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T08:16:27,010 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:27,011 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-08T08:16:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T08:16:27,012 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:27,013 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:27,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:27,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/005c5cc11d3a4da2afef57717d8635dd 2024-12-08T08:16:27,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/f68be60fd38c401fb07fbf756bb07b33 is 50, key is test_row_0/C:col10/1733645786203/Put/seqid=0 2024-12-08T08:16:27,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742343_1519 (size=12301) 2024-12-08T08:16:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T08:16:27,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T08:16:27,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:27,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:27,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:27,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:27,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:27,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T08:16:27,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:27,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:27,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:27,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:27,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T08:16:27,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/f68be60fd38c401fb07fbf756bb07b33 2024-12-08T08:16:27,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/cd1271604c054968ac1996580b3822b4 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cd1271604c054968ac1996580b3822b4 2024-12-08T08:16:27,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cd1271604c054968ac1996580b3822b4, entries=250, sequenceid=330, filesize=16.8 K 2024-12-08T08:16:27,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/005c5cc11d3a4da2afef57717d8635dd as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/005c5cc11d3a4da2afef57717d8635dd 2024-12-08T08:16:27,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:27,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/005c5cc11d3a4da2afef57717d8635dd, entries=150, sequenceid=330, filesize=12.0 K 2024-12-08T08:16:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645847461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:27,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645847461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:27,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645847461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:27,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T08:16:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:27,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:27,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:27,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:27,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:27,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:27,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/f68be60fd38c401fb07fbf756bb07b33 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/f68be60fd38c401fb07fbf756bb07b33 2024-12-08T08:16:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:27,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/f68be60fd38c401fb07fbf756bb07b33, entries=150, sequenceid=330, filesize=12.0 K 2024-12-08T08:16:27,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 92943a79353f8896400eee756d87f758 in 1242ms, sequenceid=330, compaction requested=true 2024-12-08T08:16:27,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:27,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:27,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:27,473 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:27,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:27,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:27,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:27,473 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:27,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:27,474 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:27,474 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:27,474 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/B is initiating minor compaction (all files) 2024-12-08T08:16:27,474 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/A is initiating minor compaction (all files) 2024-12-08T08:16:27,475 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/B in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:27,475 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/A in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:27,475 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/98394bd14c744dbcb09a4401d52520b4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/53dd035c000948f99a2396b70209fe5d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/005c5cc11d3a4da2afef57717d8635dd] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=36.7 K 2024-12-08T08:16:27,475 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/49bb4a687afb474dad4bda6eabb11406, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/d55f9dff3b174d8cb6b8a4c4262af0ad, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cd1271604c054968ac1996580b3822b4] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=41.5 K 2024-12-08T08:16:27,475 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 98394bd14c744dbcb09a4401d52520b4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733645783364 2024-12-08T08:16:27,475 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49bb4a687afb474dad4bda6eabb11406, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733645783364 2024-12-08T08:16:27,475 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 53dd035c000948f99a2396b70209fe5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733645784036 2024-12-08T08:16:27,475 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting d55f9dff3b174d8cb6b8a4c4262af0ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733645784036 2024-12-08T08:16:27,475 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 005c5cc11d3a4da2afef57717d8635dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733645786203 2024-12-08T08:16:27,475 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd1271604c054968ac1996580b3822b4, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733645786196 2024-12-08T08:16:27,484 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#B#compaction#444 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:27,485 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/04ba3d262e0440eda36e9bd908bd067b is 50, key is test_row_0/B:col10/1733645786203/Put/seqid=0 2024-12-08T08:16:27,485 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#A#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:27,485 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/6896b3ea2b0c46c68caa020ea1e38c75 is 50, key is test_row_0/A:col10/1733645786203/Put/seqid=0 2024-12-08T08:16:27,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742344_1520 (size=13085) 2024-12-08T08:16:27,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742345_1521 (size=13085) 2024-12-08T08:16:27,501 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/6896b3ea2b0c46c68caa020ea1e38c75 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/6896b3ea2b0c46c68caa020ea1e38c75 2024-12-08T08:16:27,505 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/A of 92943a79353f8896400eee756d87f758 into 6896b3ea2b0c46c68caa020ea1e38c75(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:27,505 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:27,505 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/A, priority=13, startTime=1733645787473; duration=0sec 2024-12-08T08:16:27,505 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:27,505 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:A 2024-12-08T08:16:27,505 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:27,506 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:27,506 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/C is initiating minor compaction (all files) 2024-12-08T08:16:27,506 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/C in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:27,506 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/36123c96cea04fdbb785048d49d80862, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/10c2c628235b4194b366143d25e37263, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/f68be60fd38c401fb07fbf756bb07b33] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=36.7 K 2024-12-08T08:16:27,506 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36123c96cea04fdbb785048d49d80862, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733645783364 2024-12-08T08:16:27,507 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10c2c628235b4194b366143d25e37263, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733645784036 2024-12-08T08:16:27,507 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting f68be60fd38c401fb07fbf756bb07b33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733645786203 2024-12-08T08:16:27,512 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#C#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:27,512 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/577b3d4f96fd4b8c9a9070c3fe4e3654 is 50, key is test_row_0/C:col10/1733645786203/Put/seqid=0 2024-12-08T08:16:27,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742346_1522 (size=13085) 2024-12-08T08:16:27,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T08:16:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:27,622 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-08T08:16:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:27,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:27,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/9d91556a444f437bb13ca68a6579b926 is 50, key is test_row_0/A:col10/1733645786303/Put/seqid=0 2024-12-08T08:16:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T08:16:27,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742347_1523 (size=12301) 2024-12-08T08:16:27,898 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/04ba3d262e0440eda36e9bd908bd067b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/04ba3d262e0440eda36e9bd908bd067b 2024-12-08T08:16:27,902 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/B of 92943a79353f8896400eee756d87f758 into 04ba3d262e0440eda36e9bd908bd067b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:27,902 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:27,902 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/B, priority=13, startTime=1733645787473; duration=0sec 2024-12-08T08:16:27,902 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:27,902 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:B 2024-12-08T08:16:27,921 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/577b3d4f96fd4b8c9a9070c3fe4e3654 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/577b3d4f96fd4b8c9a9070c3fe4e3654 2024-12-08T08:16:27,926 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/C of 92943a79353f8896400eee756d87f758 into 577b3d4f96fd4b8c9a9070c3fe4e3654(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:27,926 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:27,926 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/C, priority=13, startTime=1733645787473; duration=0sec 2024-12-08T08:16:27,926 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:27,926 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:C 2024-12-08T08:16:28,048 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/9d91556a444f437bb13ca68a6579b926 2024-12-08T08:16:28,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/ee5bd50d4a8c4608b7949086ec72b7c6 is 50, key is test_row_0/B:col10/1733645786303/Put/seqid=0 2024-12-08T08:16:28,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742348_1524 (size=12301) 2024-12-08T08:16:28,076 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/ee5bd50d4a8c4608b7949086ec72b7c6 2024-12-08T08:16:28,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/477091bcda104111809a42c09dc267ac is 50, key is test_row_0/C:col10/1733645786303/Put/seqid=0 2024-12-08T08:16:28,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742349_1525 (size=12301) 2024-12-08T08:16:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T08:16:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:28,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:28,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645848498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:28,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645848499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:28,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645848500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:28,509 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/477091bcda104111809a42c09dc267ac 2024-12-08T08:16:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/9d91556a444f437bb13ca68a6579b926 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/9d91556a444f437bb13ca68a6579b926 2024-12-08T08:16:28,515 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/9d91556a444f437bb13ca68a6579b926, entries=150, sequenceid=366, filesize=12.0 K 2024-12-08T08:16:28,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/ee5bd50d4a8c4608b7949086ec72b7c6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/ee5bd50d4a8c4608b7949086ec72b7c6 2024-12-08T08:16:28,518 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/ee5bd50d4a8c4608b7949086ec72b7c6, entries=150, sequenceid=366, filesize=12.0 K 2024-12-08T08:16:28,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/477091bcda104111809a42c09dc267ac as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/477091bcda104111809a42c09dc267ac 2024-12-08T08:16:28,521 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/477091bcda104111809a42c09dc267ac, entries=150, sequenceid=366, filesize=12.0 K 2024-12-08T08:16:28,522 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 92943a79353f8896400eee756d87f758 in 900ms, sequenceid=366, compaction requested=false 2024-12-08T08:16:28,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:28,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:28,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-08T08:16:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-08T08:16:28,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-08T08:16:28,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5110 sec 2024-12-08T08:16:28,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.5150 sec 2024-12-08T08:16:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:28,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T08:16:28,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:28,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:28,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:28,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:28,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:28,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:28,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/0ba5171dab8c4eb29b6110381d43badb is 50, key is test_row_0/A:col10/1733645788485/Put/seqid=0 2024-12-08T08:16:28,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742350_1526 (size=12301) 2024-12-08T08:16:28,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645848661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:28,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645848663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645848663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:28,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645848781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:28,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645848781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:28,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645848781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:28,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:28,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645848990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645848990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:28,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645848991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:29,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/0ba5171dab8c4eb29b6110381d43badb 2024-12-08T08:16:29,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/b2f4215358d649888da7ee74402b86c7 is 50, key is test_row_0/B:col10/1733645788485/Put/seqid=0 2024-12-08T08:16:29,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742351_1527 (size=12301) 2024-12-08T08:16:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T08:16:29,170 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:29,171 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-08T08:16:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T08:16:29,172 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:29,173 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:29,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T08:16:29,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645849300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:29,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:29,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:29,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:29,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645849310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:29,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645849310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:29,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T08:16:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:29,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:29,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:29,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/b2f4215358d649888da7ee74402b86c7 2024-12-08T08:16:29,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/488dcee1016b453985a3aeffd5f80c39 is 50, key is test_row_0/C:col10/1733645788485/Put/seqid=0 2024-12-08T08:16:29,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742352_1528 (size=12301) 2024-12-08T08:16:29,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T08:16:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:29,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T08:16:29,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T08:16:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:29,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:29,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:29,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T08:16:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:29,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T08:16:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645849820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645849820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645849821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:29,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/488dcee1016b453985a3aeffd5f80c39 2024-12-08T08:16:29,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/0ba5171dab8c4eb29b6110381d43badb as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/0ba5171dab8c4eb29b6110381d43badb 2024-12-08T08:16:29,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/0ba5171dab8c4eb29b6110381d43badb, entries=150, sequenceid=380, filesize=12.0 K 2024-12-08T08:16:29,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/b2f4215358d649888da7ee74402b86c7 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/b2f4215358d649888da7ee74402b86c7 2024-12-08T08:16:29,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/b2f4215358d649888da7ee74402b86c7, entries=150, sequenceid=380, filesize=12.0 K 2024-12-08T08:16:29,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/488dcee1016b453985a3aeffd5f80c39 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/488dcee1016b453985a3aeffd5f80c39 2024-12-08T08:16:29,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/488dcee1016b453985a3aeffd5f80c39, entries=150, sequenceid=380, filesize=12.0 K 2024-12-08T08:16:29,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 92943a79353f8896400eee756d87f758 in 1237ms, sequenceid=380, compaction requested=true 2024-12-08T08:16:29,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:29,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:29,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:29,852 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:29,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:29,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:29,852 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:29,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:29,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:29,853 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:29,853 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:29,853 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/A is initiating minor compaction (all files) 2024-12-08T08:16:29,853 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/B is initiating minor compaction (all files) 2024-12-08T08:16:29,853 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/B in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:29,853 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/A in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:29,853 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/6896b3ea2b0c46c68caa020ea1e38c75, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/9d91556a444f437bb13ca68a6579b926, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/0ba5171dab8c4eb29b6110381d43badb] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=36.8 K 2024-12-08T08:16:29,853 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/04ba3d262e0440eda36e9bd908bd067b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/ee5bd50d4a8c4608b7949086ec72b7c6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/b2f4215358d649888da7ee74402b86c7] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=36.8 K 2024-12-08T08:16:29,853 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6896b3ea2b0c46c68caa020ea1e38c75, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733645786203 2024-12-08T08:16:29,853 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 04ba3d262e0440eda36e9bd908bd067b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733645786203 2024-12-08T08:16:29,854 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5bd50d4a8c4608b7949086ec72b7c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733645786291 2024-12-08T08:16:29,854 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d91556a444f437bb13ca68a6579b926, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733645786291 2024-12-08T08:16:29,854 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f4215358d649888da7ee74402b86c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733645788485 2024-12-08T08:16:29,854 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ba5171dab8c4eb29b6110381d43badb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733645788485 2024-12-08T08:16:29,859 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:29,859 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#A#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:29,860 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/6cec00a991e848b091132a0609130355 is 50, key is test_row_0/B:col10/1733645788485/Put/seqid=0 2024-12-08T08:16:29,860 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/0d20ea609f7048458693f5c72f28fe0c is 50, key is test_row_0/A:col10/1733645788485/Put/seqid=0 2024-12-08T08:16:29,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742353_1529 (size=13187) 2024-12-08T08:16:29,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742354_1530 (size=13187) 2024-12-08T08:16:29,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T08:16:29,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:29,934 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T08:16:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/af6f9802a07247b6928895c2acb8bb52 is 50, key is test_row_0/A:col10/1733645788661/Put/seqid=0 2024-12-08T08:16:29,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742355_1531 (size=12301) 2024-12-08T08:16:30,269 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/0d20ea609f7048458693f5c72f28fe0c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/0d20ea609f7048458693f5c72f28fe0c 2024-12-08T08:16:30,269 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/6cec00a991e848b091132a0609130355 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/6cec00a991e848b091132a0609130355 2024-12-08T08:16:30,272 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/A of 92943a79353f8896400eee756d87f758 into 0d20ea609f7048458693f5c72f28fe0c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:30,272 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/B of 92943a79353f8896400eee756d87f758 into 6cec00a991e848b091132a0609130355(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:30,272 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:30,272 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:30,272 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/A, priority=13, startTime=1733645789851; duration=0sec 2024-12-08T08:16:30,272 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/B, priority=13, startTime=1733645789852; duration=0sec 2024-12-08T08:16:30,273 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:30,273 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:A 2024-12-08T08:16:30,273 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:30,273 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:B 2024-12-08T08:16:30,273 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:30,273 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:30,273 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/C is initiating minor compaction (all files) 2024-12-08T08:16:30,274 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/C in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:30,274 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/577b3d4f96fd4b8c9a9070c3fe4e3654, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/477091bcda104111809a42c09dc267ac, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/488dcee1016b453985a3aeffd5f80c39] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=36.8 K 2024-12-08T08:16:30,274 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 577b3d4f96fd4b8c9a9070c3fe4e3654, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733645786203 2024-12-08T08:16:30,274 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 477091bcda104111809a42c09dc267ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733645786291 2024-12-08T08:16:30,274 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 488dcee1016b453985a3aeffd5f80c39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733645788485 2024-12-08T08:16:30,279 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#C#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:30,279 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/a1335aba0a3c46468dd4837f21fab87e is 50, key is test_row_0/C:col10/1733645788485/Put/seqid=0 2024-12-08T08:16:30,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742356_1532 (size=13187) 2024-12-08T08:16:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T08:16:30,341 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/af6f9802a07247b6928895c2acb8bb52 2024-12-08T08:16:30,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/e7e1edbae0904f249210903c87b33695 is 50, key is test_row_0/B:col10/1733645788661/Put/seqid=0 2024-12-08T08:16:30,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742357_1533 (size=12301) 2024-12-08T08:16:30,355 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/e7e1edbae0904f249210903c87b33695 2024-12-08T08:16:30,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/6c446b8081e145abaad08b81e3beb705 is 50, key is test_row_0/C:col10/1733645788661/Put/seqid=0 2024-12-08T08:16:30,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742358_1534 (size=12301) 2024-12-08T08:16:30,375 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/6c446b8081e145abaad08b81e3beb705 2024-12-08T08:16:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/af6f9802a07247b6928895c2acb8bb52 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/af6f9802a07247b6928895c2acb8bb52 2024-12-08T08:16:30,385 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/af6f9802a07247b6928895c2acb8bb52, entries=150, sequenceid=400, filesize=12.0 K 2024-12-08T08:16:30,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/e7e1edbae0904f249210903c87b33695 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/e7e1edbae0904f249210903c87b33695 2024-12-08T08:16:30,393 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/e7e1edbae0904f249210903c87b33695, entries=150, sequenceid=400, filesize=12.0 K 2024-12-08T08:16:30,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/6c446b8081e145abaad08b81e3beb705 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/6c446b8081e145abaad08b81e3beb705 2024-12-08T08:16:30,400 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/6c446b8081e145abaad08b81e3beb705, entries=150, sequenceid=400, filesize=12.0 K 2024-12-08T08:16:30,401 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 92943a79353f8896400eee756d87f758 in 467ms, sequenceid=400, compaction requested=false 2024-12-08T08:16:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-08T08:16:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-08T08:16:30,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-08T08:16:30,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2300 sec 2024-12-08T08:16:30,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.2340 sec 2024-12-08T08:16:30,407 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T08:16:30,692 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/a1335aba0a3c46468dd4837f21fab87e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/a1335aba0a3c46468dd4837f21fab87e 2024-12-08T08:16:30,696 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/C of 92943a79353f8896400eee756d87f758 into a1335aba0a3c46468dd4837f21fab87e(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:30,696 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:30,696 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/C, priority=13, startTime=1733645789852; duration=0sec 2024-12-08T08:16:30,697 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:30,697 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:C 2024-12-08T08:16:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:30,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:16:30,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:30,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:30,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:30,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:30,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:30,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:30,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/cce396525505495d9aa37dfc280d6869 is 50, key is test_row_0/A:col10/1733645790850/Put/seqid=0 2024-12-08T08:16:30,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742359_1535 (size=14741) 2024-12-08T08:16:30,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645850894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:30,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645790889, max timestamp=1733645790900Num uniques 859; 2024-12-08T08:16:30,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:16:30,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645790889, max timestamp=1733645790900Num uniques 859; 2024-12-08T08:16:30,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:16:30,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733645790889, max timestamp=1733645790900Num uniques 859; 2024-12-08T08:16:30,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T08:16:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:30,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645850973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:30,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645850984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:31,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645851011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:31,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:31,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645851091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:31,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:31,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645851110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:31,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:31,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645851230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:31,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/cce396525505495d9aa37dfc280d6869 2024-12-08T08:16:31,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/3c8c9a5ca6a6475b969581b1df828054 is 50, key is test_row_0/B:col10/1733645790850/Put/seqid=0 2024-12-08T08:16:31,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742360_1536 (size=12301) 2024-12-08T08:16:31,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/3c8c9a5ca6a6475b969581b1df828054 2024-12-08T08:16:31,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645851301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T08:16:31,321 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:31,321 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-08T08:16:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T08:16:31,324 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:31,325 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:31,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:31,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/0fae82e59ff14b1cbe5bf2f355ccce93 is 50, key is test_row_0/C:col10/1733645790850/Put/seqid=0 2024-12-08T08:16:31,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645851321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:31,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742361_1537 (size=12301) 2024-12-08T08:16:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T08:16:31,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T08:16:31,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:31,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:31,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:31,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:31,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:31,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38480 deadline: 1733645851550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:31,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38430 deadline: 1733645851620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:31,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T08:16:31,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:31,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:31,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:31,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:31,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:31,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38512 deadline: 1733645851630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 , the old value is region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., hostname=422e41b75614,38619,1733645641981, seqNum=2 because the exception is null or not the one we care about 2024-12-08T08:16:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T08:16:31,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/0fae82e59ff14b1cbe5bf2f355ccce93 2024-12-08T08:16:31,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/cce396525505495d9aa37dfc280d6869 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cce396525505495d9aa37dfc280d6869 2024-12-08T08:16:31,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cce396525505495d9aa37dfc280d6869, entries=200, sequenceid=413, filesize=14.4 K 2024-12-08T08:16:31,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/3c8c9a5ca6a6475b969581b1df828054 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/3c8c9a5ca6a6475b969581b1df828054 2024-12-08T08:16:31,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/3c8c9a5ca6a6475b969581b1df828054, entries=150, sequenceid=413, filesize=12.0 K 2024-12-08T08:16:31,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/0fae82e59ff14b1cbe5bf2f355ccce93 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/0fae82e59ff14b1cbe5bf2f355ccce93 2024-12-08T08:16:31,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/0fae82e59ff14b1cbe5bf2f355ccce93, entries=150, sequenceid=413, filesize=12.0 K 2024-12-08T08:16:31,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for 92943a79353f8896400eee756d87f758 in 916ms, sequenceid=413, compaction requested=true 2024-12-08T08:16:31,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:31,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:31,768 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:31,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:31,768 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:31,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:31,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:31,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92943a79353f8896400eee756d87f758:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:31,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:31,769 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:31,769 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:31,769 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/B is initiating minor compaction (all files) 2024-12-08T08:16:31,769 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/A is initiating minor compaction (all files) 2024-12-08T08:16:31,769 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/A in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:31,769 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/B in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:31,769 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/0d20ea609f7048458693f5c72f28fe0c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/af6f9802a07247b6928895c2acb8bb52, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cce396525505495d9aa37dfc280d6869] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=39.3 K 2024-12-08T08:16:31,769 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/6cec00a991e848b091132a0609130355, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/e7e1edbae0904f249210903c87b33695, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/3c8c9a5ca6a6475b969581b1df828054] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=36.9 K 2024-12-08T08:16:31,769 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d20ea609f7048458693f5c72f28fe0c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733645788485 2024-12-08T08:16:31,769 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cec00a991e848b091132a0609130355, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733645788485 2024-12-08T08:16:31,769 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting e7e1edbae0904f249210903c87b33695, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733645788650 2024-12-08T08:16:31,769 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting af6f9802a07247b6928895c2acb8bb52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733645788650 2024-12-08T08:16:31,770 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting cce396525505495d9aa37dfc280d6869, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733645790841 2024-12-08T08:16:31,770 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c8c9a5ca6a6475b969581b1df828054, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733645790845 2024-12-08T08:16:31,776 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#A#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:31,776 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#B#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:31,776 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/74837df06576458bb5a7a79e64009056 is 50, key is test_row_0/A:col10/1733645790850/Put/seqid=0 2024-12-08T08:16:31,776 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/957cbb17673244af8f89074c8d6e8f94 is 50, key is test_row_0/B:col10/1733645790850/Put/seqid=0 2024-12-08T08:16:31,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742362_1538 (size=13289) 2024-12-08T08:16:31,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742363_1539 (size=13289) 2024-12-08T08:16:31,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T08:16:31,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:31,784 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-12-08T08:16:31,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:31,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:31,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:31,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:31,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:31,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:31,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/5ab0ebe86e354ffb88cbeb2bb365fb57 is 50, key is test_row_0/A:col10/1733645790982/Put/seqid=0 2024-12-08T08:16:31,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742364_1540 (size=12301) 2024-12-08T08:16:31,948 INFO [Thread-2040 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2040. 2024-12-08T08:16:31,948 DEBUG [Thread-2040 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:16:31,948 DEBUG [Thread-2040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:31,950 DEBUG [Thread-2040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:31,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:16:31,950 INFO [Thread-2036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2036. 2024-12-08T08:16:31,950 DEBUG [Thread-2036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:16:31,950 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:31,950 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T08:16:31,951 INFO [Thread-2034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2034. 2024-12-08T08:16:31,951 DEBUG [Thread-2034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:16:31,951 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:31,951 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:31,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:16:31,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:16:31,958 INFO [Thread-2038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2038. 2024-12-08T08:16:31,958 DEBUG [Thread-2038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:16:31,958 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:31,958 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:31,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:16:31,959 INFO [Thread-2032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2032. 2024-12-08T08:16:31,959 DEBUG [Thread-2032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:16:31,959 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:31,959 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:31,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:16:32,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 92943a79353f8896400eee756d87f758 2024-12-08T08:16:32,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. as already flushing 2024-12-08T08:16:32,075 INFO [Thread-2023 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2023. 2024-12-08T08:16:32,075 DEBUG [Thread-2023 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:16:32,075 DEBUG [Thread-2023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:32,075 DEBUG [Thread-2023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:32,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:16:32,132 INFO [Thread-2025 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2025. 2024-12-08T08:16:32,132 DEBUG [Thread-2025 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:16:32,132 DEBUG [Thread-2025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:32,132 DEBUG [Thread-2025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:32,133 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:16:32,153 INFO [Thread-2029 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2029. 2024-12-08T08:16:32,153 DEBUG [Thread-2029 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:16:32,153 DEBUG [Thread-2029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:32,154 DEBUG [Thread-2029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:32,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:16:32,190 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/74837df06576458bb5a7a79e64009056 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/74837df06576458bb5a7a79e64009056 2024-12-08T08:16:32,190 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/957cbb17673244af8f89074c8d6e8f94 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/957cbb17673244af8f89074c8d6e8f94 2024-12-08T08:16:32,194 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/B of 92943a79353f8896400eee756d87f758 into 957cbb17673244af8f89074c8d6e8f94(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:32,194 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/A of 92943a79353f8896400eee756d87f758 into 74837df06576458bb5a7a79e64009056(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:32,194 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:32,194 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:32,194 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/B, priority=13, startTime=1733645791768; duration=0sec 2024-12-08T08:16:32,194 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/A, priority=13, startTime=1733645791768; duration=0sec 2024-12-08T08:16:32,194 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:32,194 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:32,194 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:A 2024-12-08T08:16:32,194 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:B 2024-12-08T08:16:32,194 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:32,195 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:32,195 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 92943a79353f8896400eee756d87f758/C is initiating minor compaction (all files) 2024-12-08T08:16:32,195 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92943a79353f8896400eee756d87f758/C in TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:32,195 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/a1335aba0a3c46468dd4837f21fab87e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/6c446b8081e145abaad08b81e3beb705, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/0fae82e59ff14b1cbe5bf2f355ccce93] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp, totalSize=36.9 K 2024-12-08T08:16:32,195 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/5ab0ebe86e354ffb88cbeb2bb365fb57 2024-12-08T08:16:32,195 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1335aba0a3c46468dd4837f21fab87e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733645788485 2024-12-08T08:16:32,196 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c446b8081e145abaad08b81e3beb705, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733645788650 2024-12-08T08:16:32,196 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fae82e59ff14b1cbe5bf2f355ccce93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733645790845 2024-12-08T08:16:32,201 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92943a79353f8896400eee756d87f758#C#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/c8d386e8bb444a0d966787b28b314119 is 50, key is test_row_0/B:col10/1733645790982/Put/seqid=0 2024-12-08T08:16:32,201 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/cd97b91e46db4440899bfc321274912e is 50, key is test_row_0/C:col10/1733645790850/Put/seqid=0 2024-12-08T08:16:32,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742365_1541 (size=12301) 2024-12-08T08:16:32,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742366_1542 (size=13289) 2024-12-08T08:16:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T08:16:32,605 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/c8d386e8bb444a0d966787b28b314119 2024-12-08T08:16:32,614 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/cd97b91e46db4440899bfc321274912e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/cd97b91e46db4440899bfc321274912e 2024-12-08T08:16:32,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/d946bcd26042423cafc3d4aa60b33cb3 is 50, key is test_row_0/C:col10/1733645790982/Put/seqid=0 2024-12-08T08:16:32,618 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92943a79353f8896400eee756d87f758/C of 92943a79353f8896400eee756d87f758 into cd97b91e46db4440899bfc321274912e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:32,618 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:32,618 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., storeName=92943a79353f8896400eee756d87f758/C, priority=13, startTime=1733645791768; duration=0sec 2024-12-08T08:16:32,618 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:32,618 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92943a79353f8896400eee756d87f758:C 2024-12-08T08:16:32,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742367_1543 (size=12301) 2024-12-08T08:16:33,020 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/d946bcd26042423cafc3d4aa60b33cb3 2024-12-08T08:16:33,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/5ab0ebe86e354ffb88cbeb2bb365fb57 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5ab0ebe86e354ffb88cbeb2bb365fb57 2024-12-08T08:16:33,034 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5ab0ebe86e354ffb88cbeb2bb365fb57, entries=150, sequenceid=450, filesize=12.0 K 2024-12-08T08:16:33,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/c8d386e8bb444a0d966787b28b314119 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c8d386e8bb444a0d966787b28b314119 2024-12-08T08:16:33,040 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c8d386e8bb444a0d966787b28b314119, entries=150, sequenceid=450, filesize=12.0 K 2024-12-08T08:16:33,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/d946bcd26042423cafc3d4aa60b33cb3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/d946bcd26042423cafc3d4aa60b33cb3 2024-12-08T08:16:33,046 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/d946bcd26042423cafc3d4aa60b33cb3, entries=150, sequenceid=450, filesize=12.0 K 2024-12-08T08:16:33,047 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=20.13 KB/20610 for 92943a79353f8896400eee756d87f758 in 1263ms, sequenceid=450, compaction requested=false 2024-12-08T08:16:33,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 92943a79353f8896400eee756d87f758: 2024-12-08T08:16:33,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:33,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-08T08:16:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-08T08:16:33,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-08T08:16:33,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7240 sec 2024-12-08T08:16:33,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.7310 sec 2024-12-08T08:16:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T08:16:33,471 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:36,413 INFO [Thread-2027 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2027. 2024-12-08T08:16:36,413 DEBUG [Thread-2027 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:16:36,413 DEBUG [Thread-2027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:36,413 DEBUG [Thread-2027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:36,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:16:36,422 INFO [Thread-2021 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2021. 2024-12-08T08:16:36,422 DEBUG [Thread-2021 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:16:36,422 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:36,422 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:36,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:16:36,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T08:16:36,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-08T08:16:36,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-08T08:16:36,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-12-08T08:16:36,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-12-08T08:16:36,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-12-08T08:16:36,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T08:16:36,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T08:16:36,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1513 2024-12-08T08:16:36,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4537 rows 2024-12-08T08:16:36,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1524 2024-12-08T08:16:36,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4567 rows 2024-12-08T08:16:36,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1509 2024-12-08T08:16:36,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4527 rows 2024-12-08T08:16:36,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1512 2024-12-08T08:16:36,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4528 rows 2024-12-08T08:16:36,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1517 2024-12-08T08:16:36,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4551 rows 2024-12-08T08:16:36,423 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T08:16:36,423 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T08:16:36,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:36,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:36,423 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:16:36,424 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-08T08:16:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T08:16:36,427 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645796427"}]},"ts":"1733645796427"} 2024-12-08T08:16:36,428 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T08:16:36,428 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T08:16:36,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T08:16:36,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92943a79353f8896400eee756d87f758, UNASSIGN}] 2024-12-08T08:16:36,431 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92943a79353f8896400eee756d87f758, UNASSIGN 2024-12-08T08:16:36,432 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=92943a79353f8896400eee756d87f758, regionState=CLOSING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:16:36,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=92943a79353f8896400eee756d87f758, UNASSIGN because future has completed 2024-12-08T08:16:36,433 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T08:16:36,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:16:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T08:16:36,587 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 92943a79353f8896400eee756d87f758 2024-12-08T08:16:36,587 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T08:16:36,587 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 92943a79353f8896400eee756d87f758, disabling compactions & flushes 2024-12-08T08:16:36,587 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:36,588 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:36,588 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. after waiting 0 ms 2024-12-08T08:16:36,588 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:36,588 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing 92943a79353f8896400eee756d87f758 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T08:16:36,589 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=A 2024-12-08T08:16:36,589 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:36,589 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=B 2024-12-08T08:16:36,589 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:36,589 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 92943a79353f8896400eee756d87f758, store=C 2024-12-08T08:16:36,589 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:36,598 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/3015eb905a9f4422bc4aa91845748292 is 50, key is test_row_0/A:col10/1733645796421/Put/seqid=0 2024-12-08T08:16:36,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742368_1544 (size=12301) 2024-12-08T08:16:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T08:16:37,003 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/3015eb905a9f4422bc4aa91845748292 2024-12-08T08:16:37,010 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/c5cf51278e0245d0a123204ffc1865aa is 50, key is test_row_0/B:col10/1733645796421/Put/seqid=0 2024-12-08T08:16:37,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742369_1545 (size=12301) 2024-12-08T08:16:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T08:16:37,415 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/c5cf51278e0245d0a123204ffc1865aa 2024-12-08T08:16:37,426 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/2c126b76dda945b78c9bd071261fff72 is 50, key is test_row_0/C:col10/1733645796421/Put/seqid=0 2024-12-08T08:16:37,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742370_1546 (size=12301) 2024-12-08T08:16:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T08:16:37,832 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/2c126b76dda945b78c9bd071261fff72 2024-12-08T08:16:37,843 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/A/3015eb905a9f4422bc4aa91845748292 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/3015eb905a9f4422bc4aa91845748292 2024-12-08T08:16:37,847 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/3015eb905a9f4422bc4aa91845748292, entries=150, sequenceid=461, filesize=12.0 K 2024-12-08T08:16:37,847 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/B/c5cf51278e0245d0a123204ffc1865aa as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c5cf51278e0245d0a123204ffc1865aa 2024-12-08T08:16:37,851 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c5cf51278e0245d0a123204ffc1865aa, entries=150, sequenceid=461, filesize=12.0 K 2024-12-08T08:16:37,852 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/.tmp/C/2c126b76dda945b78c9bd071261fff72 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/2c126b76dda945b78c9bd071261fff72 2024-12-08T08:16:37,854 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/2c126b76dda945b78c9bd071261fff72, entries=150, sequenceid=461, filesize=12.0 K 2024-12-08T08:16:37,855 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 92943a79353f8896400eee756d87f758 in 1267ms, sequenceid=461, compaction requested=true 2024-12-08T08:16:37,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/57ac564612714ada9efe9c44ce4c743b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/6e8d0134c8c0451f891cebcedbd47298, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/42ad33b596044da6acdf5c6b43ebb4d6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5b18bcf1249844109a6f954de971dbed, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5412320783d64eda841ecb0c308cb90c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ff5e167972594eb8a822723339b12d10, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/8f235f25bcf24bfaa7ff9b8272ae21a6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/53e6105f068d405ba3f730995c55432b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/c3443fcce7d6464d8af54228b51ad77f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ca368e0e17134f4292a8e1449c52881a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/2de9699af3ad45edbdeca08fd6de81ad, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/e520a73e8b3347bca5b91b55aba73dfa, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/79827dc05ee548d69560ebe50c33d765, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/7e56afbcd0a34afca0d8f077e9957acf, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ac66f282c6224eea8bc8c0821a2c6157, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/1faf1d1f85004b5f8142299a1ad4f9be, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/3d6a1adba8ec4e0883851225ffc03559, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cf524a96902449d686595291c86b1250, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/8685735bb53b47fc9b3ac68c1d60a1c7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/089c993fde384cac9c4e7e4789b75de0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/49bb4a687afb474dad4bda6eabb11406, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/d55f9dff3b174d8cb6b8a4c4262af0ad, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cd1271604c054968ac1996580b3822b4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/6896b3ea2b0c46c68caa020ea1e38c75, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/9d91556a444f437bb13ca68a6579b926, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/0d20ea609f7048458693f5c72f28fe0c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/0ba5171dab8c4eb29b6110381d43badb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/af6f9802a07247b6928895c2acb8bb52, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cce396525505495d9aa37dfc280d6869] to archive 2024-12-08T08:16:37,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:16:37,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/57ac564612714ada9efe9c44ce4c743b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/57ac564612714ada9efe9c44ce4c743b 2024-12-08T08:16:37,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/6e8d0134c8c0451f891cebcedbd47298 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/6e8d0134c8c0451f891cebcedbd47298 2024-12-08T08:16:37,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/42ad33b596044da6acdf5c6b43ebb4d6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/42ad33b596044da6acdf5c6b43ebb4d6 2024-12-08T08:16:37,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5b18bcf1249844109a6f954de971dbed to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5b18bcf1249844109a6f954de971dbed 2024-12-08T08:16:37,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5412320783d64eda841ecb0c308cb90c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5412320783d64eda841ecb0c308cb90c 2024-12-08T08:16:37,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ff5e167972594eb8a822723339b12d10 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ff5e167972594eb8a822723339b12d10 2024-12-08T08:16:37,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/8f235f25bcf24bfaa7ff9b8272ae21a6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/8f235f25bcf24bfaa7ff9b8272ae21a6 2024-12-08T08:16:37,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/53e6105f068d405ba3f730995c55432b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/53e6105f068d405ba3f730995c55432b 2024-12-08T08:16:37,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/c3443fcce7d6464d8af54228b51ad77f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/c3443fcce7d6464d8af54228b51ad77f 2024-12-08T08:16:37,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ca368e0e17134f4292a8e1449c52881a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ca368e0e17134f4292a8e1449c52881a 2024-12-08T08:16:37,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/2de9699af3ad45edbdeca08fd6de81ad to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/2de9699af3ad45edbdeca08fd6de81ad 2024-12-08T08:16:37,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/e520a73e8b3347bca5b91b55aba73dfa to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/e520a73e8b3347bca5b91b55aba73dfa 2024-12-08T08:16:37,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/79827dc05ee548d69560ebe50c33d765 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/79827dc05ee548d69560ebe50c33d765 2024-12-08T08:16:37,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/7e56afbcd0a34afca0d8f077e9957acf to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/7e56afbcd0a34afca0d8f077e9957acf 2024-12-08T08:16:37,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ac66f282c6224eea8bc8c0821a2c6157 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/ac66f282c6224eea8bc8c0821a2c6157 2024-12-08T08:16:37,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/1faf1d1f85004b5f8142299a1ad4f9be to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/1faf1d1f85004b5f8142299a1ad4f9be 2024-12-08T08:16:37,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/3d6a1adba8ec4e0883851225ffc03559 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/3d6a1adba8ec4e0883851225ffc03559 2024-12-08T08:16:37,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cf524a96902449d686595291c86b1250 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cf524a96902449d686595291c86b1250 2024-12-08T08:16:37,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/8685735bb53b47fc9b3ac68c1d60a1c7 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/8685735bb53b47fc9b3ac68c1d60a1c7 2024-12-08T08:16:37,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/089c993fde384cac9c4e7e4789b75de0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/089c993fde384cac9c4e7e4789b75de0 2024-12-08T08:16:37,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/49bb4a687afb474dad4bda6eabb11406 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/49bb4a687afb474dad4bda6eabb11406 2024-12-08T08:16:37,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/d55f9dff3b174d8cb6b8a4c4262af0ad to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/d55f9dff3b174d8cb6b8a4c4262af0ad 2024-12-08T08:16:37,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cd1271604c054968ac1996580b3822b4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cd1271604c054968ac1996580b3822b4 2024-12-08T08:16:37,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/6896b3ea2b0c46c68caa020ea1e38c75 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/6896b3ea2b0c46c68caa020ea1e38c75 2024-12-08T08:16:37,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/9d91556a444f437bb13ca68a6579b926 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/9d91556a444f437bb13ca68a6579b926 2024-12-08T08:16:37,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/0d20ea609f7048458693f5c72f28fe0c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/0d20ea609f7048458693f5c72f28fe0c 2024-12-08T08:16:37,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/0ba5171dab8c4eb29b6110381d43badb to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/0ba5171dab8c4eb29b6110381d43badb 2024-12-08T08:16:37,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/af6f9802a07247b6928895c2acb8bb52 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/af6f9802a07247b6928895c2acb8bb52 2024-12-08T08:16:37,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cce396525505495d9aa37dfc280d6869 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/cce396525505495d9aa37dfc280d6869 2024-12-08T08:16:37,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/36ebea7146664b229b9af8afe018128b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c7f000c4c2364c98a512683eaf5215f1, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2e43bf1b698e4664a7be43f166d7898a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c33555425c174babb37c053640bac70e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/1d655e7756ea4c2a93ab2bbc96e0519e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2c94800bbf544a53984c9e262c93bf57, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/47db4e546a68472d8e06e5d343caddf7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2877aa8e11a543e6a41ad996f3d0a45d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2543b98bb5384accbefc24cdac7bdec6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/1a497ef63a414e2fbe9b14e8de313994, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/7bcc8ff18f5c4373afdff365fb8a8cb3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9f14665d52de4cbfacd5571a24f65129, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/bbebf197e0b14813afdd956d22d74edc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/7bc30169500e402d8a56388092598e87, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9a713ecf06f04c85bf30adaee92f2855, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/e9e9493f5d26498c84f49de8506feea3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/4be58442490c4aed89161e7046aff3af, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/0f6bca2de98a416f8ad297de285e15b5, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9ee9e684b78f4e109778b17010503d52, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/98394bd14c744dbcb09a4401d52520b4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/ded17e2cc63a4e61aa1dbaebce6a0936, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/53dd035c000948f99a2396b70209fe5d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/04ba3d262e0440eda36e9bd908bd067b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/005c5cc11d3a4da2afef57717d8635dd, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/ee5bd50d4a8c4608b7949086ec72b7c6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/6cec00a991e848b091132a0609130355, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/b2f4215358d649888da7ee74402b86c7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/e7e1edbae0904f249210903c87b33695, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/3c8c9a5ca6a6475b969581b1df828054] to archive 2024-12-08T08:16:37,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:16:37,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/36ebea7146664b229b9af8afe018128b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/36ebea7146664b229b9af8afe018128b 2024-12-08T08:16:37,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c7f000c4c2364c98a512683eaf5215f1 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c7f000c4c2364c98a512683eaf5215f1 2024-12-08T08:16:37,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2e43bf1b698e4664a7be43f166d7898a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2e43bf1b698e4664a7be43f166d7898a 2024-12-08T08:16:37,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c33555425c174babb37c053640bac70e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c33555425c174babb37c053640bac70e 2024-12-08T08:16:37,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/1d655e7756ea4c2a93ab2bbc96e0519e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/1d655e7756ea4c2a93ab2bbc96e0519e 2024-12-08T08:16:37,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2c94800bbf544a53984c9e262c93bf57 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2c94800bbf544a53984c9e262c93bf57 2024-12-08T08:16:37,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/47db4e546a68472d8e06e5d343caddf7 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/47db4e546a68472d8e06e5d343caddf7 2024-12-08T08:16:37,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2877aa8e11a543e6a41ad996f3d0a45d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2877aa8e11a543e6a41ad996f3d0a45d 2024-12-08T08:16:37,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2543b98bb5384accbefc24cdac7bdec6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/2543b98bb5384accbefc24cdac7bdec6 2024-12-08T08:16:37,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/1a497ef63a414e2fbe9b14e8de313994 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/1a497ef63a414e2fbe9b14e8de313994 2024-12-08T08:16:37,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/7bcc8ff18f5c4373afdff365fb8a8cb3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/7bcc8ff18f5c4373afdff365fb8a8cb3 2024-12-08T08:16:37,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9f14665d52de4cbfacd5571a24f65129 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9f14665d52de4cbfacd5571a24f65129 2024-12-08T08:16:37,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/bbebf197e0b14813afdd956d22d74edc to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/bbebf197e0b14813afdd956d22d74edc 2024-12-08T08:16:37,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/7bc30169500e402d8a56388092598e87 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/7bc30169500e402d8a56388092598e87 2024-12-08T08:16:37,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9a713ecf06f04c85bf30adaee92f2855 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9a713ecf06f04c85bf30adaee92f2855 2024-12-08T08:16:37,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/e9e9493f5d26498c84f49de8506feea3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/e9e9493f5d26498c84f49de8506feea3 2024-12-08T08:16:37,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/4be58442490c4aed89161e7046aff3af to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/4be58442490c4aed89161e7046aff3af 2024-12-08T08:16:37,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/0f6bca2de98a416f8ad297de285e15b5 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/0f6bca2de98a416f8ad297de285e15b5 2024-12-08T08:16:37,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9ee9e684b78f4e109778b17010503d52 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/9ee9e684b78f4e109778b17010503d52 2024-12-08T08:16:37,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/98394bd14c744dbcb09a4401d52520b4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/98394bd14c744dbcb09a4401d52520b4 2024-12-08T08:16:37,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/ded17e2cc63a4e61aa1dbaebce6a0936 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/ded17e2cc63a4e61aa1dbaebce6a0936 2024-12-08T08:16:37,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/53dd035c000948f99a2396b70209fe5d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/53dd035c000948f99a2396b70209fe5d 2024-12-08T08:16:37,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/04ba3d262e0440eda36e9bd908bd067b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/04ba3d262e0440eda36e9bd908bd067b 2024-12-08T08:16:37,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/005c5cc11d3a4da2afef57717d8635dd to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/005c5cc11d3a4da2afef57717d8635dd 2024-12-08T08:16:37,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/ee5bd50d4a8c4608b7949086ec72b7c6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/ee5bd50d4a8c4608b7949086ec72b7c6 2024-12-08T08:16:37,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/6cec00a991e848b091132a0609130355 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/6cec00a991e848b091132a0609130355 2024-12-08T08:16:37,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/b2f4215358d649888da7ee74402b86c7 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/b2f4215358d649888da7ee74402b86c7 2024-12-08T08:16:37,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/e7e1edbae0904f249210903c87b33695 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/e7e1edbae0904f249210903c87b33695 2024-12-08T08:16:37,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/3c8c9a5ca6a6475b969581b1df828054 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/3c8c9a5ca6a6475b969581b1df828054 2024-12-08T08:16:37,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e9e56dda1981448e9c80f61befe5954e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/f6c63d8cda814ea4b6765d601a78d611, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/1247e21b06c94cbcb0a397b33e17587c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e74a1a5dc8764bd8997923e274d50928, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/94d26a656d9a4dbe9ac98d0bc91d542d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/5d15c66750e74275b1ec9c400a9cae76, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/33cdd86b07564359834a6a7b413c6063, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/aec6ff7d351b4c6cbf4925007d52738b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/819ff20ca7344ccebfb6245014ebb259, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/394ea432355149fd8cc0872d4aea76fb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/bc8c1b596dc946b298d861f653a8502c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/1681f1506c7544f6a3016342ed2ee585, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/4fe99dcb442f41c39f6941f2437c4cff, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/bdf2c78cc1fb44e1ad5ad310930f575d, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/2be0c255057545499d6195149f94b739, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/4bada5794fa7426d9d8fb503fee748f7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e5b5bc5e911b4ed78201eb214d6a78be, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/c7389266c62a4f4da80623fc8c33d972, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/55ef200cc35443b19bf2fcd9188b35a9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/36123c96cea04fdbb785048d49d80862, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/3168a7f0fa164f20a1ee10daf310900c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/10c2c628235b4194b366143d25e37263, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/577b3d4f96fd4b8c9a9070c3fe4e3654, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/f68be60fd38c401fb07fbf756bb07b33, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/477091bcda104111809a42c09dc267ac, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/a1335aba0a3c46468dd4837f21fab87e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/488dcee1016b453985a3aeffd5f80c39, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/6c446b8081e145abaad08b81e3beb705, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/0fae82e59ff14b1cbe5bf2f355ccce93] to archive 2024-12-08T08:16:37,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:16:37,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e9e56dda1981448e9c80f61befe5954e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e9e56dda1981448e9c80f61befe5954e 2024-12-08T08:16:37,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/f6c63d8cda814ea4b6765d601a78d611 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/f6c63d8cda814ea4b6765d601a78d611 2024-12-08T08:16:37,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/1247e21b06c94cbcb0a397b33e17587c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/1247e21b06c94cbcb0a397b33e17587c 2024-12-08T08:16:37,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e74a1a5dc8764bd8997923e274d50928 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e74a1a5dc8764bd8997923e274d50928 2024-12-08T08:16:37,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/94d26a656d9a4dbe9ac98d0bc91d542d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/94d26a656d9a4dbe9ac98d0bc91d542d 2024-12-08T08:16:37,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/5d15c66750e74275b1ec9c400a9cae76 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/5d15c66750e74275b1ec9c400a9cae76 2024-12-08T08:16:37,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/33cdd86b07564359834a6a7b413c6063 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/33cdd86b07564359834a6a7b413c6063 2024-12-08T08:16:37,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/aec6ff7d351b4c6cbf4925007d52738b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/aec6ff7d351b4c6cbf4925007d52738b 2024-12-08T08:16:37,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/819ff20ca7344ccebfb6245014ebb259 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/819ff20ca7344ccebfb6245014ebb259 2024-12-08T08:16:37,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/394ea432355149fd8cc0872d4aea76fb to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/394ea432355149fd8cc0872d4aea76fb 2024-12-08T08:16:37,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/bc8c1b596dc946b298d861f653a8502c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/bc8c1b596dc946b298d861f653a8502c 2024-12-08T08:16:37,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/1681f1506c7544f6a3016342ed2ee585 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/1681f1506c7544f6a3016342ed2ee585 2024-12-08T08:16:37,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/4fe99dcb442f41c39f6941f2437c4cff to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/4fe99dcb442f41c39f6941f2437c4cff 2024-12-08T08:16:37,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/bdf2c78cc1fb44e1ad5ad310930f575d to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/bdf2c78cc1fb44e1ad5ad310930f575d 2024-12-08T08:16:37,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/2be0c255057545499d6195149f94b739 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/2be0c255057545499d6195149f94b739 2024-12-08T08:16:37,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/4bada5794fa7426d9d8fb503fee748f7 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/4bada5794fa7426d9d8fb503fee748f7 2024-12-08T08:16:37,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e5b5bc5e911b4ed78201eb214d6a78be to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/e5b5bc5e911b4ed78201eb214d6a78be 2024-12-08T08:16:37,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/c7389266c62a4f4da80623fc8c33d972 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/c7389266c62a4f4da80623fc8c33d972 2024-12-08T08:16:37,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/55ef200cc35443b19bf2fcd9188b35a9 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/55ef200cc35443b19bf2fcd9188b35a9 2024-12-08T08:16:37,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/36123c96cea04fdbb785048d49d80862 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/36123c96cea04fdbb785048d49d80862 2024-12-08T08:16:37,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/3168a7f0fa164f20a1ee10daf310900c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/3168a7f0fa164f20a1ee10daf310900c 2024-12-08T08:16:37,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/10c2c628235b4194b366143d25e37263 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/10c2c628235b4194b366143d25e37263 2024-12-08T08:16:37,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/577b3d4f96fd4b8c9a9070c3fe4e3654 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/577b3d4f96fd4b8c9a9070c3fe4e3654 2024-12-08T08:16:37,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/f68be60fd38c401fb07fbf756bb07b33 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/f68be60fd38c401fb07fbf756bb07b33 2024-12-08T08:16:37,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/477091bcda104111809a42c09dc267ac to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/477091bcda104111809a42c09dc267ac 2024-12-08T08:16:37,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/a1335aba0a3c46468dd4837f21fab87e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/a1335aba0a3c46468dd4837f21fab87e 2024-12-08T08:16:37,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/488dcee1016b453985a3aeffd5f80c39 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/488dcee1016b453985a3aeffd5f80c39 2024-12-08T08:16:37,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/6c446b8081e145abaad08b81e3beb705 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/6c446b8081e145abaad08b81e3beb705 2024-12-08T08:16:37,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/0fae82e59ff14b1cbe5bf2f355ccce93 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/0fae82e59ff14b1cbe5bf2f355ccce93 2024-12-08T08:16:37,921 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/recovered.edits/464.seqid, newMaxSeqId=464, maxSeqId=1 2024-12-08T08:16:37,921 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. 2024-12-08T08:16:37,922 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 92943a79353f8896400eee756d87f758: Waiting for close lock at 1733645796587Running coprocessor pre-close hooks at 1733645796587Disabling compacts and flushes for region at 1733645796587Disabling writes for close at 1733645796588 (+1 ms)Obtaining lock to block concurrent updates at 1733645796588Preparing flush snapshotting stores in 92943a79353f8896400eee756d87f758 at 1733645796588Finished memstore snapshotting TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733645796589 (+1 ms)Flushing stores of TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758. at 1733645796590 (+1 ms)Flushing 92943a79353f8896400eee756d87f758/A: creating writer at 1733645796590Flushing 92943a79353f8896400eee756d87f758/A: appending metadata at 1733645796597 (+7 ms)Flushing 92943a79353f8896400eee756d87f758/A: closing flushed file at 1733645796597Flushing 92943a79353f8896400eee756d87f758/B: creating writer at 1733645797007 (+410 ms)Flushing 92943a79353f8896400eee756d87f758/B: appending metadata at 1733645797010 (+3 ms)Flushing 92943a79353f8896400eee756d87f758/B: closing flushed file at 1733645797010Flushing 92943a79353f8896400eee756d87f758/C: creating writer at 1733645797422 (+412 ms)Flushing 92943a79353f8896400eee756d87f758/C: appending metadata at 1733645797425 (+3 ms)Flushing 92943a79353f8896400eee756d87f758/C: closing flushed file at 1733645797425Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2068af77: reopening flushed file at 1733645797842 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ada862a: reopening flushed file at 1733645797847 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40d1a684: reopening flushed file at 1733645797851 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 92943a79353f8896400eee756d87f758 in 1267ms, sequenceid=461, compaction requested=true at 1733645797855 (+4 ms)Writing region close event to WAL at 1733645797919 (+64 ms)Running coprocessor post-close hooks at 1733645797921 (+2 ms)Closed at 1733645797921 2024-12-08T08:16:37,923 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 92943a79353f8896400eee756d87f758 2024-12-08T08:16:37,923 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=92943a79353f8896400eee756d87f758, regionState=CLOSED 2024-12-08T08:16:37,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:16:37,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-08T08:16:37,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 92943a79353f8896400eee756d87f758, server=422e41b75614,38619,1733645641981 in 1.4930 sec 2024-12-08T08:16:37,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-08T08:16:37,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=92943a79353f8896400eee756d87f758, UNASSIGN in 1.4970 sec 2024-12-08T08:16:37,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-08T08:16:37,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5000 sec 2024-12-08T08:16:37,932 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645797931"}]},"ts":"1733645797931"} 2024-12-08T08:16:37,933 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T08:16:37,933 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T08:16:37,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5100 sec 2024-12-08T08:16:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T08:16:38,582 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:38,583 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-08T08:16:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:38,588 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-08T08:16:38,590 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:38,593 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758 2024-12-08T08:16:38,596 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/recovered.edits] 2024-12-08T08:16:38,601 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/3015eb905a9f4422bc4aa91845748292 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/3015eb905a9f4422bc4aa91845748292 2024-12-08T08:16:38,602 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5ab0ebe86e354ffb88cbeb2bb365fb57 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/5ab0ebe86e354ffb88cbeb2bb365fb57 2024-12-08T08:16:38,604 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/74837df06576458bb5a7a79e64009056 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/A/74837df06576458bb5a7a79e64009056 2024-12-08T08:16:38,606 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/957cbb17673244af8f89074c8d6e8f94 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/957cbb17673244af8f89074c8d6e8f94 2024-12-08T08:16:38,607 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c5cf51278e0245d0a123204ffc1865aa to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c5cf51278e0245d0a123204ffc1865aa 2024-12-08T08:16:38,609 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c8d386e8bb444a0d966787b28b314119 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/B/c8d386e8bb444a0d966787b28b314119 2024-12-08T08:16:38,611 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/2c126b76dda945b78c9bd071261fff72 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/2c126b76dda945b78c9bd071261fff72 2024-12-08T08:16:38,612 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/cd97b91e46db4440899bfc321274912e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/cd97b91e46db4440899bfc321274912e 2024-12-08T08:16:38,613 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/d946bcd26042423cafc3d4aa60b33cb3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/C/d946bcd26042423cafc3d4aa60b33cb3 2024-12-08T08:16:38,615 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/recovered.edits/464.seqid to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758/recovered.edits/464.seqid 2024-12-08T08:16:38,615 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/92943a79353f8896400eee756d87f758 2024-12-08T08:16:38,615 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T08:16:38,616 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:38,618 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T08:16:38,619 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T08:16:38,620 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:38,620 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T08:16:38,620 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733645798620"}]},"ts":"9223372036854775807"} 2024-12-08T08:16:38,622 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T08:16:38,622 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 92943a79353f8896400eee756d87f758, NAME => 'TestAcidGuarantees,,1733645769736.92943a79353f8896400eee756d87f758.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T08:16:38,622 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T08:16:38,622 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733645798622"}]},"ts":"9223372036854775807"} 2024-12-08T08:16:38,624 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T08:16:38,625 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:38,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-12-08T08:16:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-08T08:16:38,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T08:16:38,702 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:38,714 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247 (was 248), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=398 (was 404), ProcessCount=11 (was 11), AvailableMemoryMB=6964 (was 7466) 2024-12-08T08:16:38,724 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=398, ProcessCount=11, AvailableMemoryMB=6964 2024-12-08T08:16:38,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T08:16:38,726 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:16:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:38,727 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:16:38,728 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:38,728 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-12-08T08:16:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T08:16:38,728 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:16:38,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742371_1547 (size=1061) 2024-12-08T08:16:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T08:16:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T08:16:39,136 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b 2024-12-08T08:16:39,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742372_1548 (size=53) 2024-12-08T08:16:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T08:16:39,542 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:16:39,542 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 634271208c7889374f3cf3905b95dab4, disabling compactions & flushes 2024-12-08T08:16:39,542 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:39,542 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:39,542 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. after waiting 0 ms 2024-12-08T08:16:39,542 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:39,542 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:39,543 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 634271208c7889374f3cf3905b95dab4: Waiting for close lock at 1733645799542Disabling compacts and flushes for region at 1733645799542Disabling writes for close at 1733645799542Writing region close event to WAL at 1733645799542Closed at 1733645799542 2024-12-08T08:16:39,543 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:16:39,544 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733645799544"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733645799544"}]},"ts":"1733645799544"} 2024-12-08T08:16:39,545 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:16:39,546 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:16:39,546 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645799546"}]},"ts":"1733645799546"} 2024-12-08T08:16:39,548 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T08:16:39,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, ASSIGN}] 2024-12-08T08:16:39,549 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, ASSIGN 2024-12-08T08:16:39,549 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, ASSIGN; state=OFFLINE, location=422e41b75614,38619,1733645641981; forceNewPlan=false, retain=false 2024-12-08T08:16:39,700 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=634271208c7889374f3cf3905b95dab4, regionState=OPENING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:16:39,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, ASSIGN because future has completed 2024-12-08T08:16:39,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:16:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T08:16:39,909 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:39,910 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:16:39,910 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:39,911 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:16:39,911 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:39,911 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:39,914 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:39,916 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:16:39,916 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634271208c7889374f3cf3905b95dab4 columnFamilyName A 2024-12-08T08:16:39,916 DEBUG [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:39,917 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.HStore(327): Store=634271208c7889374f3cf3905b95dab4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:16:39,917 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:39,918 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:16:39,918 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634271208c7889374f3cf3905b95dab4 columnFamilyName B 2024-12-08T08:16:39,918 DEBUG [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:39,919 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.HStore(327): Store=634271208c7889374f3cf3905b95dab4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:16:39,919 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:39,920 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:16:39,921 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634271208c7889374f3cf3905b95dab4 columnFamilyName C 2024-12-08T08:16:39,921 DEBUG [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:39,921 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.HStore(327): Store=634271208c7889374f3cf3905b95dab4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:16:39,921 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:39,921 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:39,922 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:39,923 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:39,924 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:39,924 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:39,925 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T08:16:39,926 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:39,927 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:16:39,927 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 634271208c7889374f3cf3905b95dab4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59869390, jitterRate=-0.10787656903266907}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T08:16:39,927 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:39,928 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 634271208c7889374f3cf3905b95dab4: Running coprocessor pre-open hook at 1733645799911Writing region info on filesystem at 1733645799911Initializing all the Stores at 1733645799913 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645799913Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645799913Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645799914 (+1 ms)Cleaning up temporary data from old regions at 1733645799924 (+10 ms)Running coprocessor post-open hooks at 1733645799927 (+3 ms)Region opened successfully at 1733645799928 (+1 ms) 2024-12-08T08:16:39,928 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., pid=146, masterSystemTime=1733645799860 2024-12-08T08:16:39,929 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:39,929 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:39,930 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=634271208c7889374f3cf3905b95dab4, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:16:39,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:16:39,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-08T08:16:39,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 in 226 msec 2024-12-08T08:16:39,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-08T08:16:39,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, ASSIGN in 385 msec 2024-12-08T08:16:39,934 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:16:39,935 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645799934"}]},"ts":"1733645799934"} 2024-12-08T08:16:39,935 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T08:16:39,936 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:16:39,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2100 sec 2024-12-08T08:16:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T08:16:40,883 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:40,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:40,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:40,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:40,887 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c8a0c84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:40,891 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f8cd6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:40,892 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:40,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:40,895 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T08:16:40,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 422e41b75614,44803,1733645641278 2024-12-08T08:16:40,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@45f490f8 2024-12-08T08:16:40,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T08:16:40,897 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T08:16:40,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T08:16:40,899 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:16:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T08:16:40,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742373_1549 (size=1097) 2024-12-08T08:16:41,318 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T08:16:41,318 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T08:16:41,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T08:16:41,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, REOPEN/MOVE}] 2024-12-08T08:16:41,328 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, REOPEN/MOVE 2024-12-08T08:16:41,329 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=634271208c7889374f3cf3905b95dab4, regionState=CLOSING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:16:41,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, REOPEN/MOVE because future has completed 2024-12-08T08:16:41,330 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T08:16:41,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:16:41,483 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,483 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T08:16:41,483 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 634271208c7889374f3cf3905b95dab4, disabling compactions & flushes 2024-12-08T08:16:41,483 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:41,483 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:41,483 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. after waiting 0 ms 2024-12-08T08:16:41,483 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:41,486 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T08:16:41,486 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:41,486 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 634271208c7889374f3cf3905b95dab4: Waiting for close lock at 1733645801483Running coprocessor pre-close hooks at 1733645801483Disabling compacts and flushes for region at 1733645801483Disabling writes for close at 1733645801483Writing region close event to WAL at 1733645801483Running coprocessor post-close hooks at 1733645801486 (+3 ms)Closed at 1733645801486 2024-12-08T08:16:41,487 WARN [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 634271208c7889374f3cf3905b95dab4 to self. 2024-12-08T08:16:41,488 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,489 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=634271208c7889374f3cf3905b95dab4, regionState=CLOSED 2024-12-08T08:16:41,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:16:41,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-08T08:16:41,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 in 162 msec 2024-12-08T08:16:41,494 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, REOPEN/MOVE; state=CLOSED, location=422e41b75614,38619,1733645641981; forceNewPlan=false, retain=true 2024-12-08T08:16:41,597 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T08:16:41,645 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=634271208c7889374f3cf3905b95dab4, regionState=OPENING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:16:41,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, REOPEN/MOVE because future has completed 2024-12-08T08:16:41,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:16:41,810 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:41,810 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:16:41,811 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,811 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:16:41,812 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,812 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,814 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,815 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:16:41,815 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634271208c7889374f3cf3905b95dab4 columnFamilyName A 2024-12-08T08:16:41,817 DEBUG [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:41,817 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.HStore(327): Store=634271208c7889374f3cf3905b95dab4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:16:41,818 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,818 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:16:41,819 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634271208c7889374f3cf3905b95dab4 columnFamilyName B 2024-12-08T08:16:41,819 DEBUG [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:41,819 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.HStore(327): Store=634271208c7889374f3cf3905b95dab4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:16:41,819 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,820 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T08:16:41,820 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634271208c7889374f3cf3905b95dab4 columnFamilyName C 2024-12-08T08:16:41,820 DEBUG [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:41,821 INFO [StoreOpener-634271208c7889374f3cf3905b95dab4-1 {}] regionserver.HStore(327): Store=634271208c7889374f3cf3905b95dab4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:16:41,821 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:41,821 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,822 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,824 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,825 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,825 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,825 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T08:16:41,826 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,827 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 634271208c7889374f3cf3905b95dab4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64961281, jitterRate=-0.03200148046016693}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T08:16:41,827 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,828 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 634271208c7889374f3cf3905b95dab4: Running coprocessor pre-open hook at 1733645801812Writing region info on filesystem at 1733645801813 (+1 ms)Initializing all the Stores at 1733645801814 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645801814Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645801814Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733645801814Cleaning up temporary data from old regions at 1733645801825 (+11 ms)Running coprocessor post-open hooks at 1733645801827 (+2 ms)Region opened successfully at 1733645801828 (+1 ms) 2024-12-08T08:16:41,828 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., pid=151, masterSystemTime=1733645801804 2024-12-08T08:16:41,830 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:41,830 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:41,830 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=634271208c7889374f3cf3905b95dab4, regionState=OPEN, openSeqNum=5, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:16:41,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:16:41,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-12-08T08:16:41,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 in 182 msec 2024-12-08T08:16:41,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-08T08:16:41,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, REOPEN/MOVE in 506 msec 2024-12-08T08:16:41,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-08T08:16:41,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-12-08T08:16:41,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 937 msec 2024-12-08T08:16:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-08T08:16:41,838 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:41,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:41,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:41,839 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6739e27b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:41,840 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40f1a56e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:41,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:41,842 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5354df15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:41,843 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c62525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,843 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:41,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:41,844 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af4368, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:41,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:41,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:41,845 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784306f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:41,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:41,847 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e01e53e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:41,848 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e6430cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:41,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:41,850 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f49084f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:41,851 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c3aa68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:41,852 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:41,853 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bc5be61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:41,854 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ee18f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:41,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:41,855 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41db6b86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:41,856 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b7e78c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:41,858 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:41,858 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1933c978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:41,859 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f7544c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:41,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:41,860 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@588b026a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:41,861 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e37bb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,44803,-1 for getting cluster id 2024-12-08T08:16:41,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:16:41,863 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b8bced7-bfb5-4692-8aa0-6ad00e130e42' 2024-12-08T08:16:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:16:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b8bced7-bfb5-4692-8aa0-6ad00e130e42" 2024-12-08T08:16:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6e0b60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,44803,-1] 2024-12-08T08:16:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:16:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:16:41,864 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:16:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a69f325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:16:41,865 DEBUG [Thread-2464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:41,865 DEBUG [Thread-2468 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:41,865 DEBUG [Thread-2462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:41,865 DEBUG [Thread-2453 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:41,865 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:41,865 DEBUG [Thread-2460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:41,865 DEBUG [Thread-2457 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:41,865 DEBUG [Thread-2449 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:41,866 DEBUG [Thread-2466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-08T08:16:41,867 DEBUG [Thread-2451 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:41,867 DEBUG [Thread-2455 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:16:41,868 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:41,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T08:16:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:41,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:41,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:41,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:41,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:41,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,38619,1733645641981, seqNum=-1] 2024-12-08T08:16:41,869 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:16:41,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:41,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:16:41,870 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:41,870 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:16:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:16:41,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:16:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:16:41,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:16:41,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:16:41,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:16:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:16:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:16:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5] 2024-12-08T08:16:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:16:41,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:41,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:41,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:41,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:41,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:41,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:41,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645861892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:41,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645861893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:41,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645801890, max timestamp=1733645801895Num uniques 859; 2024-12-08T08:16:41,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:16:41,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645801890, max timestamp=1733645801895Num uniques 859; 2024-12-08T08:16:41,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645801890, max timestamp=1733645801895Num uniques 859; 2024-12-08T08:16:41,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:16:41,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:16:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:41,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086786b65248b447f59158cdd9dbb96f89_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645801875/Put/seqid=0 2024-12-08T08:16:41,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742374_1550 (size=12154) 2024-12-08T08:16:41,907 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:41,910 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086786b65248b447f59158cdd9dbb96f89_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086786b65248b447f59158cdd9dbb96f89_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:41,910 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/5d98e039191c40278befd281a151f32a, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:41,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/5d98e039191c40278befd281a151f32a is 175, key is test_row_0/A:col10/1733645801875/Put/seqid=0 2024-12-08T08:16:41,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742375_1551 (size=30955) 2024-12-08T08:16:41,925 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/5d98e039191c40278befd281a151f32a 2024-12-08T08:16:41,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645861922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:41,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645861923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:41,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645861923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:41,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:41,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/45bffa0553ca45a2b03a75d9881a2adf is 50, key is test_row_0/B:col10/1733645801875/Put/seqid=0 2024-12-08T08:16:41,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742376_1552 (size=12001) 2024-12-08T08:16:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T08:16:42,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645862000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645862000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T08:16:42,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:42,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:42,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:42,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:42,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:42,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645862031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645862031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645862031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T08:16:42,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:42,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:42,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:42,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:42,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T08:16:42,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645862211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645862211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645862240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645862240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645862240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T08:16:42,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:42,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:42,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:42,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:42,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:42,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/45bffa0553ca45a2b03a75d9881a2adf 2024-12-08T08:16:42,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/d249a2b374bf4a68ace156ae759de007 is 50, key is test_row_0/C:col10/1733645801875/Put/seqid=0 2024-12-08T08:16:42,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742377_1553 (size=12001) 2024-12-08T08:16:42,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/d249a2b374bf4a68ace156ae759de007 2024-12-08T08:16:42,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/5d98e039191c40278befd281a151f32a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/5d98e039191c40278befd281a151f32a 2024-12-08T08:16:42,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/5d98e039191c40278befd281a151f32a, entries=150, sequenceid=15, filesize=30.2 K 2024-12-08T08:16:42,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/45bffa0553ca45a2b03a75d9881a2adf as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/45bffa0553ca45a2b03a75d9881a2adf 2024-12-08T08:16:42,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/45bffa0553ca45a2b03a75d9881a2adf, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T08:16:42,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/d249a2b374bf4a68ace156ae759de007 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d249a2b374bf4a68ace156ae759de007 2024-12-08T08:16:42,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d249a2b374bf4a68ace156ae759de007, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T08:16:42,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 634271208c7889374f3cf3905b95dab4 in 508ms, sequenceid=15, compaction requested=false 2024-12-08T08:16:42,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:42,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T08:16:42,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:42,480 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-08T08:16:42,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:42,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:42,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:42,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:42,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:42,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:42,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084f6625e10d9e42d6b2a8817ef53902fb_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645801906/Put/seqid=0 2024-12-08T08:16:42,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742378_1554 (size=12154) 2024-12-08T08:16:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T08:16:42,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:42,503 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084f6625e10d9e42d6b2a8817ef53902fb_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084f6625e10d9e42d6b2a8817ef53902fb_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:42,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/4ed2f58a56954b4d82f31f6cdf23afcd, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:42,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/4ed2f58a56954b4d82f31f6cdf23afcd is 175, key is test_row_0/A:col10/1733645801906/Put/seqid=0 2024-12-08T08:16:42,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742379_1555 (size=30955) 2024-12-08T08:16:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:42,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:42,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645862534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645862535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645862550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645862550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645862550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645862641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645862641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645862851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645862851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:42,908 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/4ed2f58a56954b4d82f31f6cdf23afcd 2024-12-08T08:16:42,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/f8ced96bd6d0465abc1a6b91675390e6 is 50, key is test_row_0/B:col10/1733645801906/Put/seqid=0 2024-12-08T08:16:42,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742380_1556 (size=12001) 2024-12-08T08:16:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T08:16:43,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645863061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:43,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:43,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645863061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:43,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:43,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645863061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:43,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:43,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645863161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:43,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645863161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:43,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:43,317 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/f8ced96bd6d0465abc1a6b91675390e6 2024-12-08T08:16:43,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/3db736c7b6cb4c3384b10899c0bfc5b0 is 50, key is test_row_0/C:col10/1733645801906/Put/seqid=0 2024-12-08T08:16:43,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742381_1557 (size=12001) 2024-12-08T08:16:43,435 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T08:16:43,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645863670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645863671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:43,725 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/3db736c7b6cb4c3384b10899c0bfc5b0 2024-12-08T08:16:43,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/4ed2f58a56954b4d82f31f6cdf23afcd as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/4ed2f58a56954b4d82f31f6cdf23afcd 2024-12-08T08:16:43,731 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/4ed2f58a56954b4d82f31f6cdf23afcd, entries=150, sequenceid=51, filesize=30.2 K 2024-12-08T08:16:43,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/f8ced96bd6d0465abc1a6b91675390e6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/f8ced96bd6d0465abc1a6b91675390e6 2024-12-08T08:16:43,734 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/f8ced96bd6d0465abc1a6b91675390e6, entries=150, sequenceid=51, filesize=11.7 K 2024-12-08T08:16:43,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/3db736c7b6cb4c3384b10899c0bfc5b0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/3db736c7b6cb4c3384b10899c0bfc5b0 2024-12-08T08:16:43,737 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/3db736c7b6cb4c3384b10899c0bfc5b0, entries=150, sequenceid=51, filesize=11.7 K 2024-12-08T08:16:43,738 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 634271208c7889374f3cf3905b95dab4 in 1258ms, sequenceid=51, compaction requested=false 2024-12-08T08:16:43,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:43,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:43,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-08T08:16:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-08T08:16:43,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-08T08:16:43,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8700 sec 2024-12-08T08:16:43,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.8760 sec 2024-12-08T08:16:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T08:16:44,010 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:44,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-08T08:16:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T08:16:44,012 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:44,012 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:44,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:44,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T08:16:44,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:44,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:44,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:44,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:44,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:44,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:44,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087f7cf72cd2ce4d37a8462112e6de656a_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645804072/Put/seqid=0 2024-12-08T08:16:44,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742382_1558 (size=12154) 2024-12-08T08:16:44,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645864093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645864094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645864095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T08:16:44,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T08:16:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:44,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645864201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645864201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645864201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T08:16:44,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:44,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T08:16:44,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645864411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645864411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645864411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T08:16:44,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:44,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,483 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:44,486 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087f7cf72cd2ce4d37a8462112e6de656a_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087f7cf72cd2ce4d37a8462112e6de656a_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:44,486 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/19be24715895410da3b0c76d1ab07da4, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:44,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/19be24715895410da3b0c76d1ab07da4 is 175, key is test_row_0/A:col10/1733645804072/Put/seqid=0 2024-12-08T08:16:44,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742383_1559 (size=30955) 2024-12-08T08:16:44,490 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/19be24715895410da3b0c76d1ab07da4 2024-12-08T08:16:44,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/1cd143815f6e456986dc0d8545021210 is 50, key is test_row_0/B:col10/1733645804072/Put/seqid=0 2024-12-08T08:16:44,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742384_1560 (size=12001) 2024-12-08T08:16:44,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T08:16:44,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:44,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T08:16:44,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645864690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645864690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,692 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 634271208c7889374f3cf3905b95dab4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:16:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,692 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 634271208c7889374f3cf3905b95dab4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:16:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645864721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645864721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645864721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:44,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T08:16:44,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:44,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/1cd143815f6e456986dc0d8545021210 2024-12-08T08:16:44,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T08:16:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:44,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/9ca1b785cf5844a799daa32bb21ec423 is 50, key is test_row_0/C:col10/1733645804072/Put/seqid=0 2024-12-08T08:16:44,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742385_1561 (size=12001) 2024-12-08T08:16:44,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/9ca1b785cf5844a799daa32bb21ec423 2024-12-08T08:16:44,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/19be24715895410da3b0c76d1ab07da4 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/19be24715895410da3b0c76d1ab07da4 2024-12-08T08:16:44,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/19be24715895410da3b0c76d1ab07da4, entries=150, sequenceid=64, filesize=30.2 K 2024-12-08T08:16:44,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/1cd143815f6e456986dc0d8545021210 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/1cd143815f6e456986dc0d8545021210 2024-12-08T08:16:44,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/1cd143815f6e456986dc0d8545021210, entries=150, sequenceid=64, filesize=11.7 K 2024-12-08T08:16:44,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/9ca1b785cf5844a799daa32bb21ec423 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/9ca1b785cf5844a799daa32bb21ec423 2024-12-08T08:16:44,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/9ca1b785cf5844a799daa32bb21ec423, entries=150, sequenceid=64, filesize=11.7 K 2024-12-08T08:16:44,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 634271208c7889374f3cf3905b95dab4 in 900ms, sequenceid=64, compaction requested=true 2024-12-08T08:16:44,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:44,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:44,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:44,973 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:44,973 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:44,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:44,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:44,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:44,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:44,974 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:44,974 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:44,974 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/A is initiating minor compaction (all files) 2024-12-08T08:16:44,974 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/B is initiating minor compaction (all files) 2024-12-08T08:16:44,974 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/A in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,974 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/B in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,974 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/5d98e039191c40278befd281a151f32a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/4ed2f58a56954b4d82f31f6cdf23afcd, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/19be24715895410da3b0c76d1ab07da4] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=90.7 K 2024-12-08T08:16:44,974 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/45bffa0553ca45a2b03a75d9881a2adf, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/f8ced96bd6d0465abc1a6b91675390e6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/1cd143815f6e456986dc0d8545021210] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=35.2 K 2024-12-08T08:16:44,974 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:44,974 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/5d98e039191c40278befd281a151f32a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/4ed2f58a56954b4d82f31f6cdf23afcd, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/19be24715895410da3b0c76d1ab07da4] 2024-12-08T08:16:44,974 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 45bffa0553ca45a2b03a75d9881a2adf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733645801875 2024-12-08T08:16:44,974 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d98e039191c40278befd281a151f32a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733645801875 2024-12-08T08:16:44,974 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting f8ced96bd6d0465abc1a6b91675390e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733645801906 2024-12-08T08:16:44,974 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ed2f58a56954b4d82f31f6cdf23afcd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733645801906 2024-12-08T08:16:44,975 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cd143815f6e456986dc0d8545021210, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733645802526 2024-12-08T08:16:44,975 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19be24715895410da3b0c76d1ab07da4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733645802526 2024-12-08T08:16:44,979 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:44,980 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#B#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:44,981 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/db3fc306ad284e79a66d2b8715f50d81 is 50, key is test_row_0/B:col10/1733645804072/Put/seqid=0 2024-12-08T08:16:44,983 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412085cd9ab75f8734e9da9f7cf318d193174_634271208c7889374f3cf3905b95dab4 store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:44,984 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412085cd9ab75f8734e9da9f7cf318d193174_634271208c7889374f3cf3905b95dab4, store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:44,984 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085cd9ab75f8734e9da9f7cf318d193174_634271208c7889374f3cf3905b95dab4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:44,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742386_1562 (size=12104) 2024-12-08T08:16:44,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742387_1563 (size=4469) 2024-12-08T08:16:45,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T08:16:45,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:45,101 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T08:16:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:45,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082d92896c1e1f40d08ca250e87ad951f2_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645804092/Put/seqid=0 2024-12-08T08:16:45,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742388_1564 (size=12154) 2024-12-08T08:16:45,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:45,113 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082d92896c1e1f40d08ca250e87ad951f2_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082d92896c1e1f40d08ca250e87ad951f2_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:45,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/571ccd4c66f94a459181d0093add9f14, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:45,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/571ccd4c66f94a459181d0093add9f14 is 175, key is test_row_0/A:col10/1733645804092/Put/seqid=0 2024-12-08T08:16:45,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742389_1565 (size=30955) 2024-12-08T08:16:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T08:16:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:45,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:45,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645865241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:45,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645865242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:45,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:45,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:45,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:45,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645865244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:45,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645865350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:45,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645865350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645865350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:45,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:45,394 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/db3fc306ad284e79a66d2b8715f50d81 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/db3fc306ad284e79a66d2b8715f50d81 2024-12-08T08:16:45,396 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#A#compaction#480 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:45,397 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/ae8fe4e4d60440608a1a450f29525910 is 175, key is test_row_0/A:col10/1733645804072/Put/seqid=0 2024-12-08T08:16:45,399 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/B of 634271208c7889374f3cf3905b95dab4 into db3fc306ad284e79a66d2b8715f50d81(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:45,399 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:45,399 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/B, priority=13, startTime=1733645804973; duration=0sec 2024-12-08T08:16:45,399 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:45,399 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:B 2024-12-08T08:16:45,399 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:45,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742390_1566 (size=31058) 2024-12-08T08:16:45,401 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:45,401 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/C is initiating minor compaction (all files) 2024-12-08T08:16:45,401 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/C in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:45,401 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d249a2b374bf4a68ace156ae759de007, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/3db736c7b6cb4c3384b10899c0bfc5b0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/9ca1b785cf5844a799daa32bb21ec423] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=35.2 K 2024-12-08T08:16:45,402 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting d249a2b374bf4a68ace156ae759de007, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733645801875 2024-12-08T08:16:45,402 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 3db736c7b6cb4c3384b10899c0bfc5b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733645801906 2024-12-08T08:16:45,402 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ca1b785cf5844a799daa32bb21ec423, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733645802526 2024-12-08T08:16:45,404 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/ae8fe4e4d60440608a1a450f29525910 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/ae8fe4e4d60440608a1a450f29525910 2024-12-08T08:16:45,409 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#C#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:45,409 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/A of 634271208c7889374f3cf3905b95dab4 into ae8fe4e4d60440608a1a450f29525910(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:45,409 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:45,409 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/A, priority=13, startTime=1733645804973; duration=0sec 2024-12-08T08:16:45,409 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:45,409 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:A 2024-12-08T08:16:45,409 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/8c193ce2e3f84acd8d7b1cdcbb7d56dc is 50, key is test_row_0/C:col10/1733645804072/Put/seqid=0 2024-12-08T08:16:45,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742391_1567 (size=12104) 2024-12-08T08:16:45,534 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/571ccd4c66f94a459181d0093add9f14 2024-12-08T08:16:45,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/67d66acedccd459dad7e0f5d5c244d50 is 50, key is test_row_0/B:col10/1733645804092/Put/seqid=0 2024-12-08T08:16:45,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742392_1568 (size=12001) 2024-12-08T08:16:45,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645865560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:45,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645865560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:45,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:45,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645865560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:45,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:45,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:45,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:45,816 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/8c193ce2e3f84acd8d7b1cdcbb7d56dc as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/8c193ce2e3f84acd8d7b1cdcbb7d56dc 2024-12-08T08:16:45,819 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/C of 634271208c7889374f3cf3905b95dab4 into 8c193ce2e3f84acd8d7b1cdcbb7d56dc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:45,819 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:45,819 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/C, priority=13, startTime=1733645804973; duration=0sec 2024-12-08T08:16:45,819 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:45,819 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:C 2024-12-08T08:16:45,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645865870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:45,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645865871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:45,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645865871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:45,944 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/67d66acedccd459dad7e0f5d5c244d50 2024-12-08T08:16:45,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/b6b071e8b0d04ec598ddeb2f1800cde7 is 50, key is test_row_0/C:col10/1733645804092/Put/seqid=0 2024-12-08T08:16:45,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742393_1569 (size=12001) 2024-12-08T08:16:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T08:16:46,354 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/b6b071e8b0d04ec598ddeb2f1800cde7 2024-12-08T08:16:46,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/571ccd4c66f94a459181d0093add9f14 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/571ccd4c66f94a459181d0093add9f14 2024-12-08T08:16:46,361 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/571ccd4c66f94a459181d0093add9f14, entries=150, sequenceid=83, filesize=30.2 K 2024-12-08T08:16:46,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/67d66acedccd459dad7e0f5d5c244d50 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/67d66acedccd459dad7e0f5d5c244d50 2024-12-08T08:16:46,364 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/67d66acedccd459dad7e0f5d5c244d50, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T08:16:46,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/b6b071e8b0d04ec598ddeb2f1800cde7 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/b6b071e8b0d04ec598ddeb2f1800cde7 2024-12-08T08:16:46,367 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/b6b071e8b0d04ec598ddeb2f1800cde7, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T08:16:46,368 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 634271208c7889374f3cf3905b95dab4 in 1267ms, sequenceid=83, compaction requested=false 2024-12-08T08:16:46,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:46,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:46,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-08T08:16:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-08T08:16:46,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-08T08:16:46,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3560 sec 2024-12-08T08:16:46,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.3610 sec 2024-12-08T08:16:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:46,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T08:16:46,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:46,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:46,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:46,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:46,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:46,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:46,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088ec65641f64e41a09987feaf3c72dc6e_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645806382/Put/seqid=0 2024-12-08T08:16:46,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742394_1570 (size=12154) 2024-12-08T08:16:46,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645866399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:46,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645866401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645866402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:46,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645866511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:46,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645866511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645866511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:46,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645866700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:46,704 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_2' in 634271208c7889374f3cf3905b95dab4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:16:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:46,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645866711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:46,713 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 422e41b75614,38619,1733645641981 for 'test_row_1' in 634271208c7889374f3cf3905b95dab4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T08:16:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:46,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645866720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:46,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645866720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645866720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:46,799 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:46,801 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088ec65641f64e41a09987feaf3c72dc6e_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088ec65641f64e41a09987feaf3c72dc6e_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:46,802 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/dcfbfbcabd2f42739ddb70f7450b2ea2, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:46,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/dcfbfbcabd2f42739ddb70f7450b2ea2 is 175, key is test_row_0/A:col10/1733645806382/Put/seqid=0 2024-12-08T08:16:46,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742395_1571 (size=30955) 2024-12-08T08:16:47,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645867031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:47,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:47,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645867031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:47,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645867031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:47,206 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/dcfbfbcabd2f42739ddb70f7450b2ea2 2024-12-08T08:16:47,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/279df1e57ffd4a1495820abd3a6bd99f is 50, key is test_row_0/B:col10/1733645806382/Put/seqid=0 2024-12-08T08:16:47,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742396_1572 (size=12001) 2024-12-08T08:16:47,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/279df1e57ffd4a1495820abd3a6bd99f 2024-12-08T08:16:47,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/ca8312c2de15437982aac36dc1a8f9ba is 50, key is test_row_0/C:col10/1733645806382/Put/seqid=0 2024-12-08T08:16:47,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742397_1573 (size=12001) 2024-12-08T08:16:47,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645867541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:47,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645867541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:47,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645867541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:47,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/ca8312c2de15437982aac36dc1a8f9ba 2024-12-08T08:16:47,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/dcfbfbcabd2f42739ddb70f7450b2ea2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/dcfbfbcabd2f42739ddb70f7450b2ea2 2024-12-08T08:16:47,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/dcfbfbcabd2f42739ddb70f7450b2ea2, entries=150, sequenceid=101, filesize=30.2 K 2024-12-08T08:16:47,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/279df1e57ffd4a1495820abd3a6bd99f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/279df1e57ffd4a1495820abd3a6bd99f 2024-12-08T08:16:47,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/279df1e57ffd4a1495820abd3a6bd99f, entries=150, sequenceid=101, filesize=11.7 K 2024-12-08T08:16:47,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/ca8312c2de15437982aac36dc1a8f9ba as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ca8312c2de15437982aac36dc1a8f9ba 2024-12-08T08:16:47,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ca8312c2de15437982aac36dc1a8f9ba, entries=150, sequenceid=101, filesize=11.7 K 2024-12-08T08:16:47,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 634271208c7889374f3cf3905b95dab4 in 1255ms, sequenceid=101, compaction requested=true 2024-12-08T08:16:47,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:47,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:47,638 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:47,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:47,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:47,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:47,638 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:47,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:47,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:47,639 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:47,639 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:47,639 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/B is initiating minor compaction (all files) 2024-12-08T08:16:47,639 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/A is initiating minor compaction (all files) 2024-12-08T08:16:47,639 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/B in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:47,639 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/A in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:47,639 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/ae8fe4e4d60440608a1a450f29525910, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/571ccd4c66f94a459181d0093add9f14, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/dcfbfbcabd2f42739ddb70f7450b2ea2] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=90.8 K 2024-12-08T08:16:47,639 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/db3fc306ad284e79a66d2b8715f50d81, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/67d66acedccd459dad7e0f5d5c244d50, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/279df1e57ffd4a1495820abd3a6bd99f] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=35.3 K 2024-12-08T08:16:47,639 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:47,639 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/ae8fe4e4d60440608a1a450f29525910, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/571ccd4c66f94a459181d0093add9f14, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/dcfbfbcabd2f42739ddb70f7450b2ea2] 2024-12-08T08:16:47,639 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting db3fc306ad284e79a66d2b8715f50d81, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733645802526 2024-12-08T08:16:47,640 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d66acedccd459dad7e0f5d5c244d50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733645804092 2024-12-08T08:16:47,640 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae8fe4e4d60440608a1a450f29525910, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733645802526 2024-12-08T08:16:47,640 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 279df1e57ffd4a1495820abd3a6bd99f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733645805243 2024-12-08T08:16:47,640 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 571ccd4c66f94a459181d0093add9f14, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733645804092 2024-12-08T08:16:47,640 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcfbfbcabd2f42739ddb70f7450b2ea2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733645805243 2024-12-08T08:16:47,644 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:47,645 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:47,646 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/6082cf6b5e0f45a18829db5d43da3d5e is 50, key is test_row_0/B:col10/1733645806382/Put/seqid=0 2024-12-08T08:16:47,649 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208414df2c15f43457ba5495f39ad4270db_634271208c7889374f3cf3905b95dab4 store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:47,651 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208414df2c15f43457ba5495f39ad4270db_634271208c7889374f3cf3905b95dab4, store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:47,651 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208414df2c15f43457ba5495f39ad4270db_634271208c7889374f3cf3905b95dab4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:47,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742398_1574 (size=12207) 2024-12-08T08:16:47,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742399_1575 (size=4469) 2024-12-08T08:16:47,676 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#A#compaction#490 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:47,676 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/2de45d389db34b8da58a1eb2c33f16bb is 175, key is test_row_0/A:col10/1733645806382/Put/seqid=0 2024-12-08T08:16:47,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742400_1576 (size=31161) 2024-12-08T08:16:48,065 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/6082cf6b5e0f45a18829db5d43da3d5e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/6082cf6b5e0f45a18829db5d43da3d5e 2024-12-08T08:16:48,068 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/B of 634271208c7889374f3cf3905b95dab4 into 6082cf6b5e0f45a18829db5d43da3d5e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:48,068 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:48,068 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/B, priority=13, startTime=1733645807638; duration=0sec 2024-12-08T08:16:48,068 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:48,068 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:B 2024-12-08T08:16:48,068 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:48,069 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:48,069 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/C is initiating minor compaction (all files) 2024-12-08T08:16:48,069 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/C in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:48,069 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/8c193ce2e3f84acd8d7b1cdcbb7d56dc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/b6b071e8b0d04ec598ddeb2f1800cde7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ca8312c2de15437982aac36dc1a8f9ba] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=35.3 K 2024-12-08T08:16:48,069 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c193ce2e3f84acd8d7b1cdcbb7d56dc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733645802526 2024-12-08T08:16:48,069 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting b6b071e8b0d04ec598ddeb2f1800cde7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733645804092 2024-12-08T08:16:48,070 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting ca8312c2de15437982aac36dc1a8f9ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733645805243 2024-12-08T08:16:48,074 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#C#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:48,074 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/e8cffb43f7f04b79b80c1f3a9889b4b6 is 50, key is test_row_0/C:col10/1733645806382/Put/seqid=0 2024-12-08T08:16:48,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742401_1577 (size=12207) 2024-12-08T08:16:48,083 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/2de45d389db34b8da58a1eb2c33f16bb as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/2de45d389db34b8da58a1eb2c33f16bb 2024-12-08T08:16:48,086 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/A of 634271208c7889374f3cf3905b95dab4 into 2de45d389db34b8da58a1eb2c33f16bb(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:48,087 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:48,087 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/A, priority=13, startTime=1733645807638; duration=0sec 2024-12-08T08:16:48,087 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:48,087 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:A 2024-12-08T08:16:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T08:16:48,171 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:48,171 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-08T08:16:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T08:16:48,173 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:48,173 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:48,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T08:16:48,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T08:16:48,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:48,326 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T08:16:48,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:48,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:48,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:48,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:48,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:48,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:48,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208401045d2241b43d4abbdf60f92cb3b2c_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645806400/Put/seqid=0 2024-12-08T08:16:48,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742402_1578 (size=12154) 2024-12-08T08:16:48,486 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/e8cffb43f7f04b79b80c1f3a9889b4b6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/e8cffb43f7f04b79b80c1f3a9889b4b6 2024-12-08T08:16:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T08:16:48,492 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/C of 634271208c7889374f3cf3905b95dab4 into e8cffb43f7f04b79b80c1f3a9889b4b6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:48,492 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:48,492 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/C, priority=13, startTime=1733645807638; duration=0sec 2024-12-08T08:16:48,492 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:48,492 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:C 2024-12-08T08:16:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:48,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:48,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645868566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:48,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645868566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645868567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:48,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645868681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645868681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645868681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:48,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:48,738 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208401045d2241b43d4abbdf60f92cb3b2c_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208401045d2241b43d4abbdf60f92cb3b2c_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:48,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/a061bfa73d1145cea3d1a012b42565a0, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/a061bfa73d1145cea3d1a012b42565a0 is 175, key is test_row_0/A:col10/1733645806400/Put/seqid=0 2024-12-08T08:16:48,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742403_1579 (size=30955) 2024-12-08T08:16:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T08:16:48,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645868891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:48,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645868891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645868891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:49,142 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/a061bfa73d1145cea3d1a012b42565a0 2024-12-08T08:16:49,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/df11d7aa70da4570a5097d50488050de is 50, key is test_row_0/B:col10/1733645806400/Put/seqid=0 2024-12-08T08:16:49,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742404_1580 (size=12001) 2024-12-08T08:16:49,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645869201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:49,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645869201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:49,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645869201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:49,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:49,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:49,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T08:16:49,558 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/df11d7aa70da4570a5097d50488050de 2024-12-08T08:16:49,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/d215dc0f908c4b69aa5962886249320b is 50, key is test_row_0/C:col10/1733645806400/Put/seqid=0 2024-12-08T08:16:49,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742405_1581 (size=12001) 2024-12-08T08:16:49,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645869711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:49,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645869711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:49,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645869711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:49,968 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/d215dc0f908c4b69aa5962886249320b 2024-12-08T08:16:49,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/a061bfa73d1145cea3d1a012b42565a0 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/a061bfa73d1145cea3d1a012b42565a0 2024-12-08T08:16:49,974 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/a061bfa73d1145cea3d1a012b42565a0, entries=150, sequenceid=120, filesize=30.2 K 2024-12-08T08:16:49,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/df11d7aa70da4570a5097d50488050de as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/df11d7aa70da4570a5097d50488050de 2024-12-08T08:16:49,977 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/df11d7aa70da4570a5097d50488050de, entries=150, sequenceid=120, filesize=11.7 K 2024-12-08T08:16:49,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/d215dc0f908c4b69aa5962886249320b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d215dc0f908c4b69aa5962886249320b 2024-12-08T08:16:49,983 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d215dc0f908c4b69aa5962886249320b, entries=150, sequenceid=120, filesize=11.7 K 2024-12-08T08:16:49,984 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 634271208c7889374f3cf3905b95dab4 in 1658ms, sequenceid=120, compaction requested=false 2024-12-08T08:16:49,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:49,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:49,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-08T08:16:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-08T08:16:49,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-08T08:16:49,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8110 sec 2024-12-08T08:16:49,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8180 sec 2024-12-08T08:16:50,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T08:16:50,322 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:50,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:50,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-08T08:16:50,324 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:50,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T08:16:50,325 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:50,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:50,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T08:16:50,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T08:16:50,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:50,478 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T08:16:50,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:50,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:50,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:50,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:50,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:50,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:50,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208580f21b093f34077a71c18f9db31fc28_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645808565/Put/seqid=0 2024-12-08T08:16:50,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742406_1582 (size=12304) 2024-12-08T08:16:50,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:50,523 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208580f21b093f34077a71c18f9db31fc28_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208580f21b093f34077a71c18f9db31fc28_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:50,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/72419cda1d19496d9c5ece4f46683b80, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:50,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/72419cda1d19496d9c5ece4f46683b80 is 175, key is test_row_0/A:col10/1733645808565/Put/seqid=0 2024-12-08T08:16:50,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742407_1583 (size=31105) 2024-12-08T08:16:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T08:16:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:50,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:50,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:50,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645870781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645870781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:50,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645870787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:50,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645870788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:50,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645870789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:50,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645870891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:50,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645870891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:50,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645870900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:50,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645870901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:50,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645870901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T08:16:50,974 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/72419cda1d19496d9c5ece4f46683b80 2024-12-08T08:16:50,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/56b803a7d6744755bc7436b43d263fbc is 50, key is test_row_0/B:col10/1733645808565/Put/seqid=0 2024-12-08T08:16:51,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742408_1584 (size=12151) 2024-12-08T08:16:51,050 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/56b803a7d6744755bc7436b43d263fbc 2024-12-08T08:16:51,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/d403caf20c2a41e29a35830c1b90d9d2 is 50, key is test_row_0/C:col10/1733645808565/Put/seqid=0 2024-12-08T08:16:51,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645871101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645871101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645871111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645871111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645871111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742409_1585 (size=12151) 2024-12-08T08:16:51,123 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/d403caf20c2a41e29a35830c1b90d9d2 2024-12-08T08:16:51,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/72419cda1d19496d9c5ece4f46683b80 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/72419cda1d19496d9c5ece4f46683b80 2024-12-08T08:16:51,133 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/72419cda1d19496d9c5ece4f46683b80, entries=150, sequenceid=136, filesize=30.4 K 2024-12-08T08:16:51,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/56b803a7d6744755bc7436b43d263fbc as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/56b803a7d6744755bc7436b43d263fbc 2024-12-08T08:16:51,142 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/56b803a7d6744755bc7436b43d263fbc, entries=150, sequenceid=136, filesize=11.9 K 2024-12-08T08:16:51,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/d403caf20c2a41e29a35830c1b90d9d2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d403caf20c2a41e29a35830c1b90d9d2 2024-12-08T08:16:51,150 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d403caf20c2a41e29a35830c1b90d9d2, entries=150, sequenceid=136, filesize=11.9 K 2024-12-08T08:16:51,152 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 634271208c7889374f3cf3905b95dab4 in 674ms, sequenceid=136, compaction requested=true 2024-12-08T08:16:51,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:51,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:51,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-08T08:16:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-08T08:16:51,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-08T08:16:51,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 829 msec 2024-12-08T08:16:51,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 837 msec 2024-12-08T08:16:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:51,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T08:16:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:51,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208affb66ef32a84891bbc0444050679deb_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645811411/Put/seqid=0 2024-12-08T08:16:51,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645871429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645871430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645871428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645871442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645871442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T08:16:51,461 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:51,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-08T08:16:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T08:16:51,465 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:51,466 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:51,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:51,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742410_1586 (size=14794) 2024-12-08T08:16:51,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645871550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645871550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645871550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645871551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645871551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T08:16:51,596 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T08:16:51,596 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-08T08:16:51,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T08:16:51,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:51,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:51,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:51,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:51,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:51,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645871761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645871761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645871761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645871761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645871761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:51,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T08:16:51,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:51,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:51,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:51,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:51,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T08:16:51,885 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:51,888 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208affb66ef32a84891bbc0444050679deb_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208affb66ef32a84891bbc0444050679deb_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:51,889 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/7e87d0e2ae57421bb153c40e0928ec5f, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:51,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/7e87d0e2ae57421bb153c40e0928ec5f is 175, key is test_row_0/A:col10/1733645811411/Put/seqid=0 2024-12-08T08:16:51,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742411_1587 (size=39749) 2024-12-08T08:16:51,894 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/7e87d0e2ae57421bb153c40e0928ec5f 2024-12-08T08:16:51,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/2a1eddaa92224903b7e584a0b49c9fc3 is 50, key is test_row_0/B:col10/1733645811411/Put/seqid=0 2024-12-08T08:16:51,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742412_1588 (size=12151) 2024-12-08T08:16:51,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T08:16:51,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:51,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:51,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:51,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:51,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:52,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645872070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645872070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645872070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645872070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645872070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T08:16:52,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:52,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:52,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:52,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:52,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T08:16:52,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T08:16:52,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:52,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:52,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:52,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:52,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:52,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/2a1eddaa92224903b7e584a0b49c9fc3 2024-12-08T08:16:52,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/086a3b1131c24aa9b2633cacabf25c61 is 50, key is test_row_0/C:col10/1733645811411/Put/seqid=0 2024-12-08T08:16:52,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742413_1589 (size=12151) 2024-12-08T08:16:52,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/086a3b1131c24aa9b2633cacabf25c61 2024-12-08T08:16:52,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/7e87d0e2ae57421bb153c40e0928ec5f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/7e87d0e2ae57421bb153c40e0928ec5f 2024-12-08T08:16:52,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/7e87d0e2ae57421bb153c40e0928ec5f, entries=200, sequenceid=155, filesize=38.8 K 2024-12-08T08:16:52,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/2a1eddaa92224903b7e584a0b49c9fc3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/2a1eddaa92224903b7e584a0b49c9fc3 2024-12-08T08:16:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/2a1eddaa92224903b7e584a0b49c9fc3, entries=150, sequenceid=155, filesize=11.9 K 2024-12-08T08:16:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/086a3b1131c24aa9b2633cacabf25c61 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/086a3b1131c24aa9b2633cacabf25c61 2024-12-08T08:16:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/086a3b1131c24aa9b2633cacabf25c61, entries=150, sequenceid=155, filesize=11.9 K 2024-12-08T08:16:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 634271208c7889374f3cf3905b95dab4 in 925ms, sequenceid=155, compaction requested=true 2024-12-08T08:16:52,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:52,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:52,337 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:16:52,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:52,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:52,337 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:16:52,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:52,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,338 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:16:52,338 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:16:52,338 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/B is initiating minor compaction (all files) 2024-12-08T08:16:52,338 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/A is initiating minor compaction (all files) 2024-12-08T08:16:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,338 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/B in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:52,338 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/A in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:52,339 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/6082cf6b5e0f45a18829db5d43da3d5e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/df11d7aa70da4570a5097d50488050de, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/56b803a7d6744755bc7436b43d263fbc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/2a1eddaa92224903b7e584a0b49c9fc3] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=47.4 K 2024-12-08T08:16:52,339 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/2de45d389db34b8da58a1eb2c33f16bb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/a061bfa73d1145cea3d1a012b42565a0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/72419cda1d19496d9c5ece4f46683b80, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/7e87d0e2ae57421bb153c40e0928ec5f] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=129.9 K 2024-12-08T08:16:52,339 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:52,339 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/2de45d389db34b8da58a1eb2c33f16bb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/a061bfa73d1145cea3d1a012b42565a0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/72419cda1d19496d9c5ece4f46683b80, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/7e87d0e2ae57421bb153c40e0928ec5f] 2024-12-08T08:16:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,339 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 6082cf6b5e0f45a18829db5d43da3d5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733645805243 2024-12-08T08:16:52,339 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2de45d389db34b8da58a1eb2c33f16bb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733645805243 2024-12-08T08:16:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,340 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting df11d7aa70da4570a5097d50488050de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733645806397 2024-12-08T08:16:52,340 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting a061bfa73d1145cea3d1a012b42565a0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733645806397 2024-12-08T08:16:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,340 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72419cda1d19496d9c5ece4f46683b80, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733645808564 2024-12-08T08:16:52,340 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 56b803a7d6744755bc7436b43d263fbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733645808564 2024-12-08T08:16:52,340 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a1eddaa92224903b7e584a0b49c9fc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733645810786 2024-12-08T08:16:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,340 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e87d0e2ae57421bb153c40e0928ec5f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733645810779 2024-12-08T08:16:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,350 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,352 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#B#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:52,353 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/c2716353b1c64be49be5ed9ec489918f is 50, key is test_row_0/B:col10/1733645811411/Put/seqid=0 2024-12-08T08:16:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,355 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208df09a712acdf4ab8be27f0e414140cc1_634271208c7889374f3cf3905b95dab4 store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,357 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208df09a712acdf4ab8be27f0e414140cc1_634271208c7889374f3cf3905b95dab4, store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:52,357 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208df09a712acdf4ab8be27f0e414140cc1_634271208c7889374f3cf3905b95dab4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742414_1590 (size=4469) 2024-12-08T08:16:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,363 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#A#compaction#501 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,364 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/3ff8dbf5cc6240e791b1b9063e860180 is 175, key is test_row_0/A:col10/1733645811411/Put/seqid=0 2024-12-08T08:16:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742415_1591 (size=12493) 2024-12-08T08:16:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742416_1592 (size=31447) 2024-12-08T08:16:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,382 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/c2716353b1c64be49be5ed9ec489918f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/c2716353b1c64be49be5ed9ec489918f 2024-12-08T08:16:52,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T08:16:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:52,383 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T08:16:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,385 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/3ff8dbf5cc6240e791b1b9063e860180 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/3ff8dbf5cc6240e791b1b9063e860180 2024-12-08T08:16:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,389 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 634271208c7889374f3cf3905b95dab4/B of 634271208c7889374f3cf3905b95dab4 into c2716353b1c64be49be5ed9ec489918f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:52,389 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:52,389 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/B, priority=12, startTime=1733645812337; duration=0sec 2024-12-08T08:16:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,389 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:52,389 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:B 2024-12-08T08:16:52,389 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T08:16:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,390 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 634271208c7889374f3cf3905b95dab4/A of 634271208c7889374f3cf3905b95dab4 into 3ff8dbf5cc6240e791b1b9063e860180(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,390 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:52,390 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/A, priority=12, startTime=1733645812337; duration=0sec 2024-12-08T08:16:52,390 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:52,390 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:A 2024-12-08T08:16:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085ff3b9c203994d4b92f61ac01a297402_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_1/A:col10/1733645811428/Put/seqid=0 2024-12-08T08:16:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,391 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T08:16:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,391 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/C is initiating minor compaction (all files) 2024-12-08T08:16:52,391 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/C in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:52,391 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/e8cffb43f7f04b79b80c1f3a9889b4b6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d215dc0f908c4b69aa5962886249320b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d403caf20c2a41e29a35830c1b90d9d2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/086a3b1131c24aa9b2633cacabf25c61] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=47.4 K 2024-12-08T08:16:52,392 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting e8cffb43f7f04b79b80c1f3a9889b4b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733645805243 2024-12-08T08:16:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,392 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting d215dc0f908c4b69aa5962886249320b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733645806397 2024-12-08T08:16:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,393 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting d403caf20c2a41e29a35830c1b90d9d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733645808564 2024-12-08T08:16:52,393 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 086a3b1131c24aa9b2633cacabf25c61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733645810786 2024-12-08T08:16:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742417_1593 (size=9814) 2024-12-08T08:16:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,407 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#C#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,407 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/ffb0266ae4aa4df0a412a7520c41faaa is 50, key is test_row_0/C:col10/1733645811411/Put/seqid=0 2024-12-08T08:16:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,409 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085ff3b9c203994d4b92f61ac01a297402_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085ff3b9c203994d4b92f61ac01a297402_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/ea4d37ae571440ba84202f84a66866bc, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/ea4d37ae571440ba84202f84a66866bc is 175, key is test_row_1/A:col10/1733645811428/Put/seqid=0 2024-12-08T08:16:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742418_1594 (size=12493) 2024-12-08T08:16:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,427 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/ffb0266ae4aa4df0a412a7520c41faaa as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ffb0266ae4aa4df0a412a7520c41faaa 2024-12-08T08:16:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742419_1595 (size=22461) 2024-12-08T08:16:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,428 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/ea4d37ae571440ba84202f84a66866bc 2024-12-08T08:16:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,432 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 634271208c7889374f3cf3905b95dab4/C of 634271208c7889374f3cf3905b95dab4 into ffb0266ae4aa4df0a412a7520c41faaa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:52,432 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:52,432 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/C, priority=12, startTime=1733645812337; duration=0sec 2024-12-08T08:16:52,432 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:52,432 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:C 2024-12-08T08:16:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/77484a26cdca457da7cb2e0933300800 is 50, key is test_row_1/B:col10/1733645811428/Put/seqid=0 2024-12-08T08:16:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742420_1596 (size=9757) 2024-12-08T08:16:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:52,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T08:16:52,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645872603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645872604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645872604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645872605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645872607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645872721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645872721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645872721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645872721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645872721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,844 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/77484a26cdca457da7cb2e0933300800 2024-12-08T08:16:52,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/364f79784d754f85923b8144e42d3a53 is 50, key is test_row_1/C:col10/1733645811428/Put/seqid=0 2024-12-08T08:16:52,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742421_1597 (size=9757) 2024-12-08T08:16:52,872 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/364f79784d754f85923b8144e42d3a53 2024-12-08T08:16:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/ea4d37ae571440ba84202f84a66866bc as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/ea4d37ae571440ba84202f84a66866bc 2024-12-08T08:16:52,882 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/ea4d37ae571440ba84202f84a66866bc, entries=100, sequenceid=168, filesize=21.9 K 2024-12-08T08:16:52,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/77484a26cdca457da7cb2e0933300800 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/77484a26cdca457da7cb2e0933300800 2024-12-08T08:16:52,887 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/77484a26cdca457da7cb2e0933300800, entries=100, sequenceid=168, filesize=9.5 K 2024-12-08T08:16:52,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/364f79784d754f85923b8144e42d3a53 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/364f79784d754f85923b8144e42d3a53 2024-12-08T08:16:52,894 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/364f79784d754f85923b8144e42d3a53, entries=100, sequenceid=168, filesize=9.5 K 2024-12-08T08:16:52,895 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 634271208c7889374f3cf3905b95dab4 in 512ms, sequenceid=168, compaction requested=false 2024-12-08T08:16:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-08T08:16:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-08T08:16:52,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-08T08:16:52,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4310 sec 2024-12-08T08:16:52,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.4370 sec 2024-12-08T08:16:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:52,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-08T08:16:52,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:52,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:52,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:52,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:52,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:52,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:52,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208189fde7058204da390f311861d099613_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645812605/Put/seqid=0 2024-12-08T08:16:52,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645872944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645872945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645872945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645872945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645872946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:52,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742422_1598 (size=12304) 2024-12-08T08:16:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645873061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645873061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645873061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645873061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645873061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645873271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645873271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645873271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645873271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645873271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,366 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:53,369 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208189fde7058204da390f311861d099613_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208189fde7058204da390f311861d099613_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:53,369 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/1212a1f36f894d4192ab8dfc17845078, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:53,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/1212a1f36f894d4192ab8dfc17845078 is 175, key is test_row_0/A:col10/1733645812605/Put/seqid=0 2024-12-08T08:16:53,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742423_1599 (size=31105) 2024-12-08T08:16:53,373 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/1212a1f36f894d4192ab8dfc17845078 2024-12-08T08:16:53,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/d96d96c5f8b14c9eb2ff6446855e15cd is 50, key is test_row_0/B:col10/1733645812605/Put/seqid=0 2024-12-08T08:16:53,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742424_1600 (size=12151) 2024-12-08T08:16:53,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645873580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645873580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645873581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645873581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645873581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T08:16:53,621 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:53,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-08T08:16:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T08:16:53,622 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:53,623 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:53,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T08:16:53,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T08:16:53,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:53,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:53,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:53,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:53,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:53,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/d96d96c5f8b14c9eb2ff6446855e15cd 2024-12-08T08:16:53,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/221db69e1a5343969912176d26900953 is 50, key is test_row_0/C:col10/1733645812605/Put/seqid=0 2024-12-08T08:16:53,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742425_1601 (size=12151) 2024-12-08T08:16:53,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T08:16:53,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:53,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:53,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:53,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:53,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T08:16:54,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T08:16:54,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:54,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:54,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:54,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:54,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:54,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645874091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:54,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645874091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:54,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645874091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:54,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645874091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:54,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645874091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:54,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/221db69e1a5343969912176d26900953 2024-12-08T08:16:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/1212a1f36f894d4192ab8dfc17845078 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/1212a1f36f894d4192ab8dfc17845078 2024-12-08T08:16:54,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/1212a1f36f894d4192ab8dfc17845078, entries=150, sequenceid=193, filesize=30.4 K 2024-12-08T08:16:54,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/d96d96c5f8b14c9eb2ff6446855e15cd as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/d96d96c5f8b14c9eb2ff6446855e15cd 2024-12-08T08:16:54,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/d96d96c5f8b14c9eb2ff6446855e15cd, entries=150, sequenceid=193, filesize=11.9 K 2024-12-08T08:16:54,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/221db69e1a5343969912176d26900953 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/221db69e1a5343969912176d26900953 2024-12-08T08:16:54,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/221db69e1a5343969912176d26900953, entries=150, sequenceid=193, filesize=11.9 K 2024-12-08T08:16:54,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 634271208c7889374f3cf3905b95dab4 in 1283ms, sequenceid=193, compaction requested=true 2024-12-08T08:16:54,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:54,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:54,217 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:54,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:54,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:54,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:54,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:54,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T08:16:54,217 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:54,218 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:54,218 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/A is initiating minor compaction (all files) 2024-12-08T08:16:54,218 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/A in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:54,218 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/3ff8dbf5cc6240e791b1b9063e860180, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/ea4d37ae571440ba84202f84a66866bc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/1212a1f36f894d4192ab8dfc17845078] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=83.0 K 2024-12-08T08:16:54,218 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:54,218 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/3ff8dbf5cc6240e791b1b9063e860180, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/ea4d37ae571440ba84202f84a66866bc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/1212a1f36f894d4192ab8dfc17845078] 2024-12-08T08:16:54,219 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ff8dbf5cc6240e791b1b9063e860180, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733645810786 2024-12-08T08:16:54,219 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:54,219 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/B is initiating minor compaction (all files) 2024-12-08T08:16:54,219 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/B in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:54,219 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/c2716353b1c64be49be5ed9ec489918f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/77484a26cdca457da7cb2e0933300800, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/d96d96c5f8b14c9eb2ff6446855e15cd] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=33.6 K 2024-12-08T08:16:54,219 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea4d37ae571440ba84202f84a66866bc, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733645811428 2024-12-08T08:16:54,219 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting c2716353b1c64be49be5ed9ec489918f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733645810786 2024-12-08T08:16:54,219 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 77484a26cdca457da7cb2e0933300800, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733645811428 2024-12-08T08:16:54,219 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1212a1f36f894d4192ab8dfc17845078, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733645812605 2024-12-08T08:16:54,220 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting d96d96c5f8b14c9eb2ff6446855e15cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733645812605 2024-12-08T08:16:54,229 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:54,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T08:16:54,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:54,233 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:16:54,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:54,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:54,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:54,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:54,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:54,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:54,242 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#B#compaction#511 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:54,242 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/4e7f36e1ef3f44f0b8470190d1e0969e is 50, key is test_row_0/B:col10/1733645812605/Put/seqid=0 2024-12-08T08:16:54,249 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208eafb4be13a58499a87f0b568dcd065c4_634271208c7889374f3cf3905b95dab4 store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T08:16:54,251 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208eafb4be13a58499a87f0b568dcd065c4_634271208c7889374f3cf3905b95dab4, store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:54,251 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208eafb4be13a58499a87f0b568dcd065c4_634271208c7889374f3cf3905b95dab4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:54,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120821232d4f15b34470ba83b4e4b20d4608_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645812945/Put/seqid=0 2024-12-08T08:16:54,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742426_1602 (size=12595) 2024-12-08T08:16:54,282 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/4e7f36e1ef3f44f0b8470190d1e0969e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/4e7f36e1ef3f44f0b8470190d1e0969e 2024-12-08T08:16:54,288 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/B of 634271208c7889374f3cf3905b95dab4 into 4e7f36e1ef3f44f0b8470190d1e0969e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:54,288 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:54,288 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/B, priority=13, startTime=1733645814217; duration=0sec 2024-12-08T08:16:54,288 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:54,288 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:B 2024-12-08T08:16:54,288 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:54,292 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:54,292 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/C is initiating minor compaction (all files) 2024-12-08T08:16:54,292 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/C in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:54,292 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ffb0266ae4aa4df0a412a7520c41faaa, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/364f79784d754f85923b8144e42d3a53, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/221db69e1a5343969912176d26900953] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=33.6 K 2024-12-08T08:16:54,292 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting ffb0266ae4aa4df0a412a7520c41faaa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733645810786 2024-12-08T08:16:54,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742427_1603 (size=4469) 2024-12-08T08:16:54,295 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 364f79784d754f85923b8144e42d3a53, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733645811428 2024-12-08T08:16:54,295 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#A#compaction#510 average throughput is 0.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:54,295 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 221db69e1a5343969912176d26900953, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733645812605 2024-12-08T08:16:54,295 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/9165515a2d73435b941a2154fba9ee2e is 175, key is test_row_0/A:col10/1733645812605/Put/seqid=0 2024-12-08T08:16:54,312 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#C#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:54,312 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/e97a428c05604cdcb1b00f4a8ff24767 is 50, key is test_row_0/C:col10/1733645812605/Put/seqid=0 2024-12-08T08:16:54,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742430_1606 (size=12595) 2024-12-08T08:16:54,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742428_1604 (size=12304) 2024-12-08T08:16:54,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742429_1605 (size=31549) 2024-12-08T08:16:54,328 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/e97a428c05604cdcb1b00f4a8ff24767 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/e97a428c05604cdcb1b00f4a8ff24767 2024-12-08T08:16:54,332 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/9165515a2d73435b941a2154fba9ee2e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/9165515a2d73435b941a2154fba9ee2e 2024-12-08T08:16:54,332 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/C of 634271208c7889374f3cf3905b95dab4 into e97a428c05604cdcb1b00f4a8ff24767(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:54,332 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:54,332 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/C, priority=13, startTime=1733645814217; duration=0sec 2024-12-08T08:16:54,333 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:54,333 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:C 2024-12-08T08:16:54,338 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/A of 634271208c7889374f3cf3905b95dab4 into 9165515a2d73435b941a2154fba9ee2e(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:54,338 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:54,338 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/A, priority=13, startTime=1733645814217; duration=0sec 2024-12-08T08:16:54,338 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:54,338 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:A 2024-12-08T08:16:54,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:54,724 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120821232d4f15b34470ba83b4e4b20d4608_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120821232d4f15b34470ba83b4e4b20d4608_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:54,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/756b3343096c4a2eacd02b9d3c5039e9, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:54,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/756b3343096c4a2eacd02b9d3c5039e9 is 175, key is test_row_0/A:col10/1733645812945/Put/seqid=0 2024-12-08T08:16:54,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742431_1607 (size=31105) 2024-12-08T08:16:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T08:16:55,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:55,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645815120, max timestamp=1733645815125Num uniques 859; 2024-12-08T08:16:55,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645815120, max timestamp=1733645815125Num uniques 859; 2024-12-08T08:16:55,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:16:55,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:16:55,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733645815120, max timestamp=1733645815125Num uniques 859; 2024-12-08T08:16:55,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T08:16:55,128 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/756b3343096c4a2eacd02b9d3c5039e9 2024-12-08T08:16:55,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/78318de9fbb44497b767eb1456113671 is 50, key is test_row_0/B:col10/1733645812945/Put/seqid=0 2024-12-08T08:16:55,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742432_1608 (size=12151) 2024-12-08T08:16:55,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645875143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645875143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645875144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645875145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645875146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645875251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645875251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645875260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645875260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645875260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645875461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645875461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645875470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645875470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645875470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,540 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/78318de9fbb44497b767eb1456113671 2024-12-08T08:16:55,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/ec4a549f975e4186a0dea0f2ba510ff2 is 50, key is test_row_0/C:col10/1733645812945/Put/seqid=0 2024-12-08T08:16:55,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742433_1609 (size=12151) 2024-12-08T08:16:55,551 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/ec4a549f975e4186a0dea0f2ba510ff2 2024-12-08T08:16:55,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/756b3343096c4a2eacd02b9d3c5039e9 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/756b3343096c4a2eacd02b9d3c5039e9 2024-12-08T08:16:55,558 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/756b3343096c4a2eacd02b9d3c5039e9, entries=150, sequenceid=203, filesize=30.4 K 2024-12-08T08:16:55,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/78318de9fbb44497b767eb1456113671 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/78318de9fbb44497b767eb1456113671 2024-12-08T08:16:55,563 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/78318de9fbb44497b767eb1456113671, entries=150, sequenceid=203, filesize=11.9 K 2024-12-08T08:16:55,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/ec4a549f975e4186a0dea0f2ba510ff2 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ec4a549f975e4186a0dea0f2ba510ff2 2024-12-08T08:16:55,566 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ec4a549f975e4186a0dea0f2ba510ff2, entries=150, sequenceid=203, filesize=11.9 K 2024-12-08T08:16:55,567 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 634271208c7889374f3cf3905b95dab4 in 1334ms, sequenceid=203, compaction requested=false 2024-12-08T08:16:55,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:55,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:55,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-08T08:16:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-08T08:16:55,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-08T08:16:55,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9460 sec 2024-12-08T08:16:55,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.9500 sec 2024-12-08T08:16:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T08:16:55,771 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:55,772 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-08T08:16:55,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-08T08:16:55,774 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:55,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T08:16:55,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:55,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:55,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:55,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:55,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T08:16:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:55,776 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:55,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:16:55,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645875782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082270dd07845a477299c6435ebce2b047_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645815145/Put/seqid=0 2024-12-08T08:16:55,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645875785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645875790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645875790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645875792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742434_1610 (size=14794) 2024-12-08T08:16:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T08:16:55,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645875900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645875900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645875900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645875900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645875900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:55,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-08T08:16:55,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:55,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:55,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:55,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:55,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:56,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-08T08:16:56,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:56,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:56,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:56,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:56,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T08:16:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645876110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645876110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645876110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645876111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645876111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,210 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:56,213 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082270dd07845a477299c6435ebce2b047_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082270dd07845a477299c6435ebce2b047_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:56,216 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/c337b00a8eec4473862452d5a11cb136, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:56,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/c337b00a8eec4473862452d5a11cb136 is 175, key is test_row_0/A:col10/1733645815145/Put/seqid=0 2024-12-08T08:16:56,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742435_1611 (size=39749) 2024-12-08T08:16:56,220 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/c337b00a8eec4473862452d5a11cb136 2024-12-08T08:16:56,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/684b71c669394fae82740f40bfad0a41 is 50, key is test_row_0/B:col10/1733645815145/Put/seqid=0 2024-12-08T08:16:56,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742436_1612 (size=12151) 2024-12-08T08:16:56,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/684b71c669394fae82740f40bfad0a41 2024-12-08T08:16:56,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-08T08:16:56,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:56,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:56,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:56,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:56,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:16:56,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/410b00ee6da94402a044b07074c04fac is 50, key is test_row_0/C:col10/1733645815145/Put/seqid=0 2024-12-08T08:16:56,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742437_1613 (size=12151) 2024-12-08T08:16:56,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/410b00ee6da94402a044b07074c04fac 2024-12-08T08:16:56,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/c337b00a8eec4473862452d5a11cb136 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/c337b00a8eec4473862452d5a11cb136 2024-12-08T08:16:56,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/c337b00a8eec4473862452d5a11cb136, entries=200, sequenceid=243, filesize=38.8 K 2024-12-08T08:16:56,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/684b71c669394fae82740f40bfad0a41 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/684b71c669394fae82740f40bfad0a41 2024-12-08T08:16:56,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/684b71c669394fae82740f40bfad0a41, entries=150, sequenceid=243, filesize=11.9 K 2024-12-08T08:16:56,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/410b00ee6da94402a044b07074c04fac as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/410b00ee6da94402a044b07074c04fac 2024-12-08T08:16:56,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/410b00ee6da94402a044b07074c04fac, entries=150, sequenceid=243, filesize=11.9 K 2024-12-08T08:16:56,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 634271208c7889374f3cf3905b95dab4 in 498ms, sequenceid=243, compaction requested=true 2024-12-08T08:16:56,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:56,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:16:56,273 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:56,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:56,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:16:56,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:56,273 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:56,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:16:56,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:56,273 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:56,273 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:56,274 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/A is initiating minor compaction (all files) 2024-12-08T08:16:56,274 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/B is initiating minor compaction (all files) 2024-12-08T08:16:56,274 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/B in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:56,274 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/A in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:56,274 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/9165515a2d73435b941a2154fba9ee2e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/756b3343096c4a2eacd02b9d3c5039e9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/c337b00a8eec4473862452d5a11cb136] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=100.0 K 2024-12-08T08:16:56,274 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/4e7f36e1ef3f44f0b8470190d1e0969e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/78318de9fbb44497b767eb1456113671, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/684b71c669394fae82740f40bfad0a41] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=36.0 K 2024-12-08T08:16:56,274 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:56,274 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/9165515a2d73435b941a2154fba9ee2e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/756b3343096c4a2eacd02b9d3c5039e9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/c337b00a8eec4473862452d5a11cb136] 2024-12-08T08:16:56,274 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e7f36e1ef3f44f0b8470190d1e0969e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733645812605 2024-12-08T08:16:56,274 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9165515a2d73435b941a2154fba9ee2e, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733645812605 2024-12-08T08:16:56,274 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 78318de9fbb44497b767eb1456113671, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733645812942 2024-12-08T08:16:56,274 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 756b3343096c4a2eacd02b9d3c5039e9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733645812942 2024-12-08T08:16:56,274 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 684b71c669394fae82740f40bfad0a41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733645815142 2024-12-08T08:16:56,274 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting c337b00a8eec4473862452d5a11cb136, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733645815142 2024-12-08T08:16:56,279 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:56,280 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#B#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:56,281 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/59b52f0f7a4845a2bd28521361a61156 is 50, key is test_row_0/B:col10/1733645815145/Put/seqid=0 2024-12-08T08:16:56,281 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208cff35f7ea0934dfb8274f0a1f071b2bf_634271208c7889374f3cf3905b95dab4 store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:56,283 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208cff35f7ea0934dfb8274f0a1f071b2bf_634271208c7889374f3cf3905b95dab4, store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:56,283 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208cff35f7ea0934dfb8274f0a1f071b2bf_634271208c7889374f3cf3905b95dab4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:56,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742438_1614 (size=12697) 2024-12-08T08:16:56,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742439_1615 (size=4469) 2024-12-08T08:16:56,312 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#A#compaction#520 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:56,312 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/9e13ffb136724c95b64a7b789830f5b3 is 175, key is test_row_0/A:col10/1733645815145/Put/seqid=0 2024-12-08T08:16:56,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742440_1616 (size=31651) 2024-12-08T08:16:56,324 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/9e13ffb136724c95b64a7b789830f5b3 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/9e13ffb136724c95b64a7b789830f5b3 2024-12-08T08:16:56,328 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/A of 634271208c7889374f3cf3905b95dab4 into 9e13ffb136724c95b64a7b789830f5b3(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:56,328 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:56,328 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/A, priority=13, startTime=1733645816272; duration=0sec 2024-12-08T08:16:56,328 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:16:56,328 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:A 2024-12-08T08:16:56,328 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:16:56,329 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:16:56,329 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/C is initiating minor compaction (all files) 2024-12-08T08:16:56,329 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/C in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:56,330 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/e97a428c05604cdcb1b00f4a8ff24767, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ec4a549f975e4186a0dea0f2ba510ff2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/410b00ee6da94402a044b07074c04fac] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=36.0 K 2024-12-08T08:16:56,330 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting e97a428c05604cdcb1b00f4a8ff24767, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733645812605 2024-12-08T08:16:56,330 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec4a549f975e4186a0dea0f2ba510ff2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733645812942 2024-12-08T08:16:56,330 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 410b00ee6da94402a044b07074c04fac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733645815142 2024-12-08T08:16:56,337 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#C#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:16:56,337 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/0f4ac45423194146b96eb1c88910fd5c is 50, key is test_row_0/C:col10/1733645815145/Put/seqid=0 2024-12-08T08:16:56,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742441_1617 (size=12697) 2024-12-08T08:16:56,359 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/0f4ac45423194146b96eb1c88910fd5c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/0f4ac45423194146b96eb1c88910fd5c 2024-12-08T08:16:56,364 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/C of 634271208c7889374f3cf3905b95dab4 into 0f4ac45423194146b96eb1c88910fd5c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:56,364 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:56,364 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/C, priority=13, startTime=1733645816273; duration=0sec 2024-12-08T08:16:56,364 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:56,364 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:C 2024-12-08T08:16:56,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-08T08:16:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:56,388 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:16:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:56,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:56,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:56,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:56,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:56,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:16:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T08:16:56,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e05d6893d0b64666b2ab310b5def3536_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645815787/Put/seqid=0 2024-12-08T08:16:56,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742442_1618 (size=12304) 2024-12-08T08:16:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:56,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:16:56,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645816436, max timestamp=1733645816442Num uniques 859; 2024-12-08T08:16:56,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645816436, max timestamp=1733645816442Num uniques 859; 2024-12-08T08:16:56,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733645816436, max timestamp=1733645816442Num uniques 859; 2024-12-08T08:16:56,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:16:56,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:16:56,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T08:16:56,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645876461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645876462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645876463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645876464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645876465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645876570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645876570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645876570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645876570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645876581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,699 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/59b52f0f7a4845a2bd28521361a61156 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/59b52f0f7a4845a2bd28521361a61156 2024-12-08T08:16:56,702 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/B of 634271208c7889374f3cf3905b95dab4 into 59b52f0f7a4845a2bd28521361a61156(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:16:56,702 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:56,702 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/B, priority=13, startTime=1733645816273; duration=0sec 2024-12-08T08:16:56,702 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:16:56,702 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:B 2024-12-08T08:16:56,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645876780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645876780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645876780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645876780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645876791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:56,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:56,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:56,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:56,815 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e05d6893d0b64666b2ab310b5def3536_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e05d6893d0b64666b2ab310b5def3536_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/992b3f512c4748d08e71bef41823e00b, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:56,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/992b3f512c4748d08e71bef41823e00b is 175, key is test_row_0/A:col10/1733645815787/Put/seqid=0 2024-12-08T08:16:56,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742443_1619 (size=31105) 2024-12-08T08:16:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T08:16:57,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645877090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:57,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645877090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645877090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:57,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645877090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:57,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645877101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:57,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:57,219 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/992b3f512c4748d08e71bef41823e00b 2024-12-08T08:16:57,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/26d5b3e47127463a92069cb65327d6cf is 50, key is test_row_0/B:col10/1733645815787/Put/seqid=0 2024-12-08T08:16:57,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742444_1620 (size=12151) 2024-12-08T08:16:57,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645877600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:57,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645877600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645877600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:57,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645877600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:57,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645877611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:57,627 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/26d5b3e47127463a92069cb65327d6cf 2024-12-08T08:16:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/b05e465e3a3740a6a9af457c792aad3e is 50, key is test_row_0/C:col10/1733645815787/Put/seqid=0 2024-12-08T08:16:57,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742445_1621 (size=12151) 2024-12-08T08:16:57,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T08:16:58,036 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/b05e465e3a3740a6a9af457c792aad3e 2024-12-08T08:16:58,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/992b3f512c4748d08e71bef41823e00b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/992b3f512c4748d08e71bef41823e00b 2024-12-08T08:16:58,043 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/992b3f512c4748d08e71bef41823e00b, entries=150, sequenceid=255, filesize=30.4 K 2024-12-08T08:16:58,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/26d5b3e47127463a92069cb65327d6cf as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/26d5b3e47127463a92069cb65327d6cf 2024-12-08T08:16:58,047 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/26d5b3e47127463a92069cb65327d6cf, entries=150, sequenceid=255, filesize=11.9 K 2024-12-08T08:16:58,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/b05e465e3a3740a6a9af457c792aad3e as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/b05e465e3a3740a6a9af457c792aad3e 2024-12-08T08:16:58,051 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/b05e465e3a3740a6a9af457c792aad3e, entries=150, sequenceid=255, filesize=11.9 K 2024-12-08T08:16:58,052 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 634271208c7889374f3cf3905b95dab4 in 1663ms, sequenceid=255, compaction requested=false 2024-12-08T08:16:58,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:16:58,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:16:58,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-08T08:16:58,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-08T08:16:58,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-08T08:16:58,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2770 sec 2024-12-08T08:16:58,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.2830 sec 2024-12-08T08:16:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:58,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-08T08:16:58,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:16:58,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T08:16:58,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:16:58,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T08:16:58,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:16:58,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T08:16:58,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208583cdce18ee84689872714daee1d77f1_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645816461/Put/seqid=0 2024-12-08T08:16:58,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742446_1622 (size=14994) 2024-12-08T08:16:58,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645878624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645878625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645878625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645878625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645878626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645878731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645878730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645878731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645878731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645878740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645878940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645878941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645878941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645878941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:58,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645878950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:59,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:16:59,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208583cdce18ee84689872714daee1d77f1_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208583cdce18ee84689872714daee1d77f1_634271208c7889374f3cf3905b95dab4 2024-12-08T08:16:59,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/bac1b6305be348a79869c39749841981, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:16:59,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/bac1b6305be348a79869c39749841981 is 175, key is test_row_0/A:col10/1733645816461/Put/seqid=0 2024-12-08T08:16:59,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742447_1623 (size=39949) 2024-12-08T08:16:59,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645879250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:59,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645879250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:59,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645879250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645879251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:59,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:59,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645879261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:59,435 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/bac1b6305be348a79869c39749841981 2024-12-08T08:16:59,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/f6e4b0474c7a44c9af434275f5064225 is 50, key is test_row_0/B:col10/1733645816461/Put/seqid=0 2024-12-08T08:16:59,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742448_1624 (size=12301) 2024-12-08T08:16:59,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645879760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:59,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645879760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:59,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:59,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645879760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:59,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:59,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:59,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645879761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:59,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:16:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645879771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:16:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:16:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:16:59,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/f6e4b0474c7a44c9af434275f5064225 2024-12-08T08:16:59,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/56f0bd7561e24558a75563240cf0412a is 50, key is test_row_0/C:col10/1733645816461/Put/seqid=0 2024-12-08T08:16:59,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742449_1625 (size=12301) 2024-12-08T08:16:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T08:16:59,930 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:16:59,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-08T08:16:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-08T08:16:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T08:16:59,932 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T08:16:59,933 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T08:16:59,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T08:17:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T08:17:00,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-08T08:17:00,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:00,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:17:00,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:00,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:17:00,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:17:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:17:00,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-08T08:17:00,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:00,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:17:00,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:00,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:17:00,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:17:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:17:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T08:17:00,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/56f0bd7561e24558a75563240cf0412a 2024-12-08T08:17:00,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/bac1b6305be348a79869c39749841981 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/bac1b6305be348a79869c39749841981 2024-12-08T08:17:00,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/bac1b6305be348a79869c39749841981, entries=200, sequenceid=294, filesize=39.0 K 2024-12-08T08:17:00,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/f6e4b0474c7a44c9af434275f5064225 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/f6e4b0474c7a44c9af434275f5064225 2024-12-08T08:17:00,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/f6e4b0474c7a44c9af434275f5064225, entries=150, sequenceid=294, filesize=12.0 K 2024-12-08T08:17:00,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/56f0bd7561e24558a75563240cf0412a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/56f0bd7561e24558a75563240cf0412a 2024-12-08T08:17:00,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/56f0bd7561e24558a75563240cf0412a, entries=150, sequenceid=294, filesize=12.0 K 2024-12-08T08:17:00,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=46.96 KB/48090 for 634271208c7889374f3cf3905b95dab4 in 1666ms, sequenceid=294, compaction requested=true 2024-12-08T08:17:00,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:17:00,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:17:00,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:17:00,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:17:00,278 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:17:00,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:17:00,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T08:17:00,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:17:00,279 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:17:00,279 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:17:00,279 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:17:00,279 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/B is initiating minor compaction (all files) 2024-12-08T08:17:00,279 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/A is initiating minor compaction (all files) 2024-12-08T08:17:00,279 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/B in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:00,279 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/A in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:00,279 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/9e13ffb136724c95b64a7b789830f5b3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/992b3f512c4748d08e71bef41823e00b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/bac1b6305be348a79869c39749841981] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=100.3 K 2024-12-08T08:17:00,279 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/59b52f0f7a4845a2bd28521361a61156, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/26d5b3e47127463a92069cb65327d6cf, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/f6e4b0474c7a44c9af434275f5064225] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=36.3 K 2024-12-08T08:17:00,279 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:00,280 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. files: [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/9e13ffb136724c95b64a7b789830f5b3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/992b3f512c4748d08e71bef41823e00b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/bac1b6305be348a79869c39749841981] 2024-12-08T08:17:00,280 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 59b52f0f7a4845a2bd28521361a61156, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733645815142 2024-12-08T08:17:00,280 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e13ffb136724c95b64a7b789830f5b3, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733645815142 2024-12-08T08:17:00,280 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 26d5b3e47127463a92069cb65327d6cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733645815783 2024-12-08T08:17:00,280 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting 992b3f512c4748d08e71bef41823e00b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733645815783 2024-12-08T08:17:00,280 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting f6e4b0474c7a44c9af434275f5064225, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733645816461 2024-12-08T08:17:00,280 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] compactions.Compactor(225): Compacting bac1b6305be348a79869c39749841981, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733645816457 2024-12-08T08:17:00,285 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:17:00,285 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#B#compaction#528 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:17:00,286 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/9e31b5f2cb5f4038b2260b7f38722606 is 50, key is test_row_0/B:col10/1733645816461/Put/seqid=0 2024-12-08T08:17:00,287 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120831b4cd2a2ee9464bbac4ec83f8cb52b9_634271208c7889374f3cf3905b95dab4 store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:17:00,288 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120831b4cd2a2ee9464bbac4ec83f8cb52b9_634271208c7889374f3cf3905b95dab4, store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:17:00,288 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120831b4cd2a2ee9464bbac4ec83f8cb52b9_634271208c7889374f3cf3905b95dab4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:17:00,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742450_1626 (size=12949) 2024-12-08T08:17:00,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742451_1627 (size=4469) 2024-12-08T08:17:00,293 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#A#compaction#529 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:17:00,293 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/febbbeb624f146eaaa2dd3b9b969845f is 175, key is test_row_0/A:col10/1733645816461/Put/seqid=0 2024-12-08T08:17:00,294 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/9e31b5f2cb5f4038b2260b7f38722606 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/9e31b5f2cb5f4038b2260b7f38722606 2024-12-08T08:17:00,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742452_1628 (size=31903) 2024-12-08T08:17:00,297 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/B of 634271208c7889374f3cf3905b95dab4 into 9e31b5f2cb5f4038b2260b7f38722606(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:17:00,297 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:17:00,297 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/B, priority=13, startTime=1733645820278; duration=0sec 2024-12-08T08:17:00,297 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T08:17:00,297 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:B 2024-12-08T08:17:00,297 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T08:17:00,298 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T08:17:00,298 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1541): 634271208c7889374f3cf3905b95dab4/C is initiating minor compaction (all files) 2024-12-08T08:17:00,298 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 634271208c7889374f3cf3905b95dab4/C in TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:00,298 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/0f4ac45423194146b96eb1c88910fd5c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/b05e465e3a3740a6a9af457c792aad3e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/56f0bd7561e24558a75563240cf0412a] into tmpdir=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp, totalSize=36.3 K 2024-12-08T08:17:00,298 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f4ac45423194146b96eb1c88910fd5c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733645815142 2024-12-08T08:17:00,299 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting b05e465e3a3740a6a9af457c792aad3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733645815783 2024-12-08T08:17:00,299 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] compactions.Compactor(225): Compacting 56f0bd7561e24558a75563240cf0412a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733645816461 2024-12-08T08:17:00,304 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 634271208c7889374f3cf3905b95dab4#C#compaction#530 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T08:17:00,304 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/5d8357c5b8784797a069d67d036a5471 is 50, key is test_row_0/C:col10/1733645816461/Put/seqid=0 2024-12-08T08:17:00,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742453_1629 (size=12949) 2024-12-08T08:17:00,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-08T08:17:00,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:00,389 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T08:17:00,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:17:00,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:17:00,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:17:00,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:17:00,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:17:00,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:17:00,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a57ad792855c43ccb80c59c1bb736c0e_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645818624/Put/seqid=0 2024-12-08T08:17:00,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742454_1630 (size=12454) 2024-12-08T08:17:00,407 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T08:17:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T08:17:00,700 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/febbbeb624f146eaaa2dd3b9b969845f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/febbbeb624f146eaaa2dd3b9b969845f 2024-12-08T08:17:00,704 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/A of 634271208c7889374f3cf3905b95dab4 into febbbeb624f146eaaa2dd3b9b969845f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:17:00,704 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:17:00,704 INFO [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/A, priority=13, startTime=1733645820278; duration=0sec 2024-12-08T08:17:00,704 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:17:00,704 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:A 2024-12-08T08:17:00,715 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/5d8357c5b8784797a069d67d036a5471 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/5d8357c5b8784797a069d67d036a5471 2024-12-08T08:17:00,718 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 634271208c7889374f3cf3905b95dab4/C of 634271208c7889374f3cf3905b95dab4 into 5d8357c5b8784797a069d67d036a5471(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T08:17:00,718 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:17:00,718 INFO [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., storeName=634271208c7889374f3cf3905b95dab4/C, priority=13, startTime=1733645820279; duration=0sec 2024-12-08T08:17:00,718 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:17:00,718 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:C 2024-12-08T08:17:00,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. as already flushing 2024-12-08T08:17:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:00,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:17:00,800 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a57ad792855c43ccb80c59c1bb736c0e_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a57ad792855c43ccb80c59c1bb736c0e_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:00,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/a607dd5a5bdf42f58c8755cd4479535f, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:17:00,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/a607dd5a5bdf42f58c8755cd4479535f is 175, key is test_row_0/A:col10/1733645818624/Put/seqid=0 2024-12-08T08:17:00,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742455_1631 (size=31255) 2024-12-08T08:17:00,805 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/a607dd5a5bdf42f58c8755cd4479535f 2024-12-08T08:17:00,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/cc3925a5e1654ce082b52b3ef802bf2a is 50, key is test_row_0/B:col10/1733645818624/Put/seqid=0 2024-12-08T08:17:00,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742456_1632 (size=12301) 2024-12-08T08:17:00,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733645820783, max timestamp=1733645820811Num uniques 859; 2024-12-08T08:17:00,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:17:00,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733645820783, max timestamp=1733645820811Num uniques 859; 2024-12-08T08:17:00,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:17:00,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733645820783, max timestamp=1733645820811Num uniques 859; 2024-12-08T08:17:00,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/422e41b75614:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T08:17:00,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645880842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:00,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645880843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:00,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645880848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645880848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645880849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:00,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645880960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645880961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:00,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645880961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:00,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645880961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:00,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645880961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T08:17:01,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:01,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645881171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645881171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:01,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645881171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:01,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645881171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:01,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645881171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:01,220 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/cc3925a5e1654ce082b52b3ef802bf2a 2024-12-08T08:17:01,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/8561ffd9003a421fbe1c36fdd83a6f10 is 50, key is test_row_0/C:col10/1733645818624/Put/seqid=0 2024-12-08T08:17:01,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742457_1633 (size=12301) 2024-12-08T08:17:01,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57820 deadline: 1733645881480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:01,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57806 deadline: 1733645881480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:01,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57802 deadline: 1733645881480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:01,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1733645881481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T08:17:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57736 deadline: 1733645881481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 2024-12-08T08:17:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 , the old value is region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T08:17:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., hostname=422e41b75614,38619,1733645641981, seqNum=5 because the exception is null or not the one we care about 2024-12-08T08:17:01,634 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/8561ffd9003a421fbe1c36fdd83a6f10 2024-12-08T08:17:01,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/a607dd5a5bdf42f58c8755cd4479535f as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/a607dd5a5bdf42f58c8755cd4479535f 2024-12-08T08:17:01,642 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/a607dd5a5bdf42f58c8755cd4479535f, entries=150, sequenceid=305, filesize=30.5 K 2024-12-08T08:17:01,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/cc3925a5e1654ce082b52b3ef802bf2a as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/cc3925a5e1654ce082b52b3ef802bf2a 2024-12-08T08:17:01,647 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/cc3925a5e1654ce082b52b3ef802bf2a, entries=150, sequenceid=305, filesize=12.0 K 2024-12-08T08:17:01,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/8561ffd9003a421fbe1c36fdd83a6f10 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/8561ffd9003a421fbe1c36fdd83a6f10 2024-12-08T08:17:01,652 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/8561ffd9003a421fbe1c36fdd83a6f10, entries=150, sequenceid=305, filesize=12.0 K 2024-12-08T08:17:01,653 INFO [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 634271208c7889374f3cf3905b95dab4 in 1264ms, sequenceid=305, compaction requested=false 2024-12-08T08:17:01,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:17:01,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:01,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-08T08:17:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-08T08:17:01,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-08T08:17:01,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7220 sec 2024-12-08T08:17:01,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.7270 sec 2024-12-08T08:17:01,868 INFO [Thread-2466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2466. 2024-12-08T08:17:01,868 DEBUG [Thread-2466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:17:01,868 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,868 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:17:01,869 INFO [Thread-2462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2462. 2024-12-08T08:17:01,869 DEBUG [Thread-2462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:17:01,869 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,870 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:17:01,870 INFO [Thread-2464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2464. 2024-12-08T08:17:01,870 DEBUG [Thread-2464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:17:01,870 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,870 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:17:01,871 INFO [Thread-2468 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2468. 2024-12-08T08:17:01,871 DEBUG [Thread-2468 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:17:01,871 DEBUG [Thread-2468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,871 DEBUG [Thread-2468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:17:01,872 INFO [Thread-2460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2460. 2024-12-08T08:17:01,873 DEBUG [Thread-2460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:17:01,873 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,873 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,873 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:17:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38619 {}] regionserver.HRegion(8855): Flush requested on 634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:01,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=161.02 KB heapSize=456 KB 2024-12-08T08:17:01,993 INFO [Thread-2451 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2451. 2024-12-08T08:17:01,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:17:01,993 DEBUG [Thread-2451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:17:01,993 INFO [Thread-2455 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2455. 2024-12-08T08:17:01,993 DEBUG [Thread-2451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,993 DEBUG [Thread-2455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:17:01,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T08:17:01,993 DEBUG [Thread-2455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,993 INFO [Thread-2449 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2449. 2024-12-08T08:17:01,993 DEBUG [Thread-2451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:17:01,993 DEBUG [Thread-2449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:17:01,993 DEBUG [Thread-2449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T08:17:01,993 DEBUG [Thread-2455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:17:01,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T08:17:01,993 DEBUG [Thread-2449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:17:01,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:17:01,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:17:01,995 INFO [Thread-2457 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2457. 2024-12-08T08:17:01,995 DEBUG [Thread-2457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:17:01,995 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,995 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,995 INFO [Thread-2453 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2453. 2024-12-08T08:17:01,995 DEBUG [Thread-2453 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T08:17:01,995 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,995 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:01,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:17:01,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:17:01,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081fb4d8dbf86940aa8680ad125b5843c6_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_0/A:col10/1733645821991/Put/seqid=0 2024-12-08T08:17:02,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742458_1634 (size=12454) 2024-12-08T08:17:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T08:17:02,081 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:17:02,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T08:17:02,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-08T08:17:02,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-08T08:17:02,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-08T08:17:02,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-08T08:17:02,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-12-08T08:17:02,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T08:17:02,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4217 2024-12-08T08:17:02,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4274 2024-12-08T08:17:02,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4253 2024-12-08T08:17:02,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4210 2024-12-08T08:17:02,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4304 2024-12-08T08:17:02,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T08:17:02,082 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T08:17:02,082 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T08:17:02,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:02,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:02,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:17:02,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-08T08:17:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T08:17:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-08T08:17:02,092 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645822092"}]},"ts":"1733645822092"} 2024-12-08T08:17:02,094 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T08:17:02,094 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T08:17:02,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T08:17:02,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, UNASSIGN}] 2024-12-08T08:17:02,097 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, UNASSIGN 2024-12-08T08:17:02,098 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=634271208c7889374f3cf3905b95dab4, regionState=CLOSING, regionLocation=422e41b75614,38619,1733645641981 2024-12-08T08:17:02,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, UNASSIGN because future has completed 2024-12-08T08:17:02,099 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T08:17:02,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981}] 2024-12-08T08:17:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-08T08:17:02,253 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close 634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:02,253 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T08:17:02,253 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing 634271208c7889374f3cf3905b95dab4, disabling compactions & flushes 2024-12-08T08:17:02,254 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:02,402 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:17:02,405 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081fb4d8dbf86940aa8680ad125b5843c6_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081fb4d8dbf86940aa8680ad125b5843c6_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:02,406 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/528eeeca552d430aa502f0fbdc97fb69, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:17:02,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/528eeeca552d430aa502f0fbdc97fb69 is 175, key is test_row_0/A:col10/1733645821991/Put/seqid=0 2024-12-08T08:17:02,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742459_1635 (size=31255) 2024-12-08T08:17:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-08T08:17:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-08T08:17:02,810 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/528eeeca552d430aa502f0fbdc97fb69 2024-12-08T08:17:02,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/92c414a9c5a340c4964542de553d39f8 is 50, key is test_row_0/B:col10/1733645821991/Put/seqid=0 2024-12-08T08:17:02,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742460_1636 (size=12301) 2024-12-08T08:17:03,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/92c414a9c5a340c4964542de553d39f8 2024-12-08T08:17:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-08T08:17:03,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/91b21d29350a4b0ba259b419d432d4fd is 50, key is test_row_0/C:col10/1733645821991/Put/seqid=0 2024-12-08T08:17:03,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742461_1637 (size=12301) 2024-12-08T08:17:03,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/91b21d29350a4b0ba259b419d432d4fd 2024-12-08T08:17:03,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/528eeeca552d430aa502f0fbdc97fb69 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/528eeeca552d430aa502f0fbdc97fb69 2024-12-08T08:17:03,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/528eeeca552d430aa502f0fbdc97fb69, entries=150, sequenceid=346, filesize=30.5 K 2024-12-08T08:17:03,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/92c414a9c5a340c4964542de553d39f8 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/92c414a9c5a340c4964542de553d39f8 2024-12-08T08:17:03,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/92c414a9c5a340c4964542de553d39f8, entries=150, sequenceid=346, filesize=12.0 K 2024-12-08T08:17:03,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/91b21d29350a4b0ba259b419d432d4fd as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/91b21d29350a4b0ba259b419d432d4fd 2024-12-08T08:17:03,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/91b21d29350a4b0ba259b419d432d4fd, entries=150, sequenceid=346, filesize=12.0 K 2024-12-08T08:17:03,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~455.95 KB/466896, currentSize=13.42 KB/13740 for 634271208c7889374f3cf3905b95dab4 in 1666ms, sequenceid=346, compaction requested=true 2024-12-08T08:17:03,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 634271208c7889374f3cf3905b95dab4: 2024-12-08T08:17:03,658 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:03,658 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:03,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T08:17:03,658 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. after waiting 0 ms 2024-12-08T08:17:03,658 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:03,658 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. because compaction request was cancelled 2024-12-08T08:17:03,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:17:03,658 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:A 2024-12-08T08:17:03,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:17:03,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:17:03,658 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing 634271208c7889374f3cf3905b95dab4 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-08T08:17:03,658 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. because compaction request was cancelled 2024-12-08T08:17:03,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 634271208c7889374f3cf3905b95dab4:C, priority=-2147483648, current under compaction store size is 2 2024-12-08T08:17:03,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T08:17:03,658 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. because compaction request was cancelled 2024-12-08T08:17:03,658 DEBUG [RS:0;422e41b75614:38619-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:B 2024-12-08T08:17:03,658 DEBUG [RS:0;422e41b75614:38619-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 634271208c7889374f3cf3905b95dab4:C 2024-12-08T08:17:03,659 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=A 2024-12-08T08:17:03,659 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:17:03,659 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=B 2024-12-08T08:17:03,659 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:17:03,659 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 634271208c7889374f3cf3905b95dab4, store=C 2024-12-08T08:17:03,659 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T08:17:03,662 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208de5404c7569f4e71b87769b2b9993dbc_634271208c7889374f3cf3905b95dab4 is 50, key is test_row_2/A:col10/1733645821994/Put/seqid=0 2024-12-08T08:17:03,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742462_1638 (size=7374) 2024-12-08T08:17:04,066 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:17:04,073 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208de5404c7569f4e71b87769b2b9993dbc_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208de5404c7569f4e71b87769b2b9993dbc_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:04,075 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/f50872a0eed44ecfb7dc76ca2a54849c, store: [table=TestAcidGuarantees family=A region=634271208c7889374f3cf3905b95dab4] 2024-12-08T08:17:04,076 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/f50872a0eed44ecfb7dc76ca2a54849c is 175, key is test_row_2/A:col10/1733645821994/Put/seqid=0 2024-12-08T08:17:04,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742463_1639 (size=13865) 2024-12-08T08:17:04,081 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/f50872a0eed44ecfb7dc76ca2a54849c 2024-12-08T08:17:04,089 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/aa35b3fadcd6419daea298534e11126c is 50, key is test_row_2/B:col10/1733645821994/Put/seqid=0 2024-12-08T08:17:04,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742464_1640 (size=7415) 2024-12-08T08:17:04,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-08T08:17:04,496 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/aa35b3fadcd6419daea298534e11126c 2024-12-08T08:17:04,509 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/9420fa84578e4ace8812cd05e83b37e9 is 50, key is test_row_2/C:col10/1733645821994/Put/seqid=0 2024-12-08T08:17:04,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742465_1641 (size=7415) 2024-12-08T08:17:04,914 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/9420fa84578e4ace8812cd05e83b37e9 2024-12-08T08:17:04,922 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/A/f50872a0eed44ecfb7dc76ca2a54849c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/f50872a0eed44ecfb7dc76ca2a54849c 2024-12-08T08:17:04,927 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/f50872a0eed44ecfb7dc76ca2a54849c, entries=50, sequenceid=351, filesize=13.5 K 2024-12-08T08:17:04,928 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/B/aa35b3fadcd6419daea298534e11126c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/aa35b3fadcd6419daea298534e11126c 2024-12-08T08:17:04,932 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/aa35b3fadcd6419daea298534e11126c, entries=50, sequenceid=351, filesize=7.2 K 2024-12-08T08:17:04,933 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/.tmp/C/9420fa84578e4ace8812cd05e83b37e9 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/9420fa84578e4ace8812cd05e83b37e9 2024-12-08T08:17:04,936 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/9420fa84578e4ace8812cd05e83b37e9, entries=50, sequenceid=351, filesize=7.2 K 2024-12-08T08:17:04,969 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 634271208c7889374f3cf3905b95dab4 in 1311ms, sequenceid=351, compaction requested=true 2024-12-08T08:17:04,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/5d98e039191c40278befd281a151f32a, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/4ed2f58a56954b4d82f31f6cdf23afcd, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/ae8fe4e4d60440608a1a450f29525910, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/19be24715895410da3b0c76d1ab07da4, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/571ccd4c66f94a459181d0093add9f14, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/2de45d389db34b8da58a1eb2c33f16bb, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/dcfbfbcabd2f42739ddb70f7450b2ea2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/a061bfa73d1145cea3d1a012b42565a0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/72419cda1d19496d9c5ece4f46683b80, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/7e87d0e2ae57421bb153c40e0928ec5f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/3ff8dbf5cc6240e791b1b9063e860180, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/ea4d37ae571440ba84202f84a66866bc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/9165515a2d73435b941a2154fba9ee2e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/1212a1f36f894d4192ab8dfc17845078, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/756b3343096c4a2eacd02b9d3c5039e9, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/c337b00a8eec4473862452d5a11cb136, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/9e13ffb136724c95b64a7b789830f5b3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/992b3f512c4748d08e71bef41823e00b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/bac1b6305be348a79869c39749841981] to archive 2024-12-08T08:17:04,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:17:04,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/5d98e039191c40278befd281a151f32a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/5d98e039191c40278befd281a151f32a 2024-12-08T08:17:04,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/4ed2f58a56954b4d82f31f6cdf23afcd to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/4ed2f58a56954b4d82f31f6cdf23afcd 2024-12-08T08:17:04,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/ae8fe4e4d60440608a1a450f29525910 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/ae8fe4e4d60440608a1a450f29525910 2024-12-08T08:17:04,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/19be24715895410da3b0c76d1ab07da4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/19be24715895410da3b0c76d1ab07da4 2024-12-08T08:17:04,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/571ccd4c66f94a459181d0093add9f14 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/571ccd4c66f94a459181d0093add9f14 2024-12-08T08:17:04,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/2de45d389db34b8da58a1eb2c33f16bb to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/2de45d389db34b8da58a1eb2c33f16bb 2024-12-08T08:17:04,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/dcfbfbcabd2f42739ddb70f7450b2ea2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/dcfbfbcabd2f42739ddb70f7450b2ea2 2024-12-08T08:17:04,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/a061bfa73d1145cea3d1a012b42565a0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/a061bfa73d1145cea3d1a012b42565a0 2024-12-08T08:17:04,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/72419cda1d19496d9c5ece4f46683b80 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/72419cda1d19496d9c5ece4f46683b80 2024-12-08T08:17:04,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/7e87d0e2ae57421bb153c40e0928ec5f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/7e87d0e2ae57421bb153c40e0928ec5f 2024-12-08T08:17:04,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/3ff8dbf5cc6240e791b1b9063e860180 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/3ff8dbf5cc6240e791b1b9063e860180 2024-12-08T08:17:04,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/ea4d37ae571440ba84202f84a66866bc to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/ea4d37ae571440ba84202f84a66866bc 2024-12-08T08:17:04,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/9165515a2d73435b941a2154fba9ee2e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/9165515a2d73435b941a2154fba9ee2e 2024-12-08T08:17:04,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/1212a1f36f894d4192ab8dfc17845078 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/1212a1f36f894d4192ab8dfc17845078 2024-12-08T08:17:04,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/756b3343096c4a2eacd02b9d3c5039e9 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/756b3343096c4a2eacd02b9d3c5039e9 2024-12-08T08:17:04,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/c337b00a8eec4473862452d5a11cb136 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/c337b00a8eec4473862452d5a11cb136 2024-12-08T08:17:04,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/9e13ffb136724c95b64a7b789830f5b3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/9e13ffb136724c95b64a7b789830f5b3 2024-12-08T08:17:04,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/992b3f512c4748d08e71bef41823e00b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/992b3f512c4748d08e71bef41823e00b 2024-12-08T08:17:04,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/bac1b6305be348a79869c39749841981 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/bac1b6305be348a79869c39749841981 2024-12-08T08:17:04,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/45bffa0553ca45a2b03a75d9881a2adf, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/f8ced96bd6d0465abc1a6b91675390e6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/db3fc306ad284e79a66d2b8715f50d81, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/1cd143815f6e456986dc0d8545021210, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/67d66acedccd459dad7e0f5d5c244d50, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/6082cf6b5e0f45a18829db5d43da3d5e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/279df1e57ffd4a1495820abd3a6bd99f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/df11d7aa70da4570a5097d50488050de, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/56b803a7d6744755bc7436b43d263fbc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/c2716353b1c64be49be5ed9ec489918f, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/2a1eddaa92224903b7e584a0b49c9fc3, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/77484a26cdca457da7cb2e0933300800, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/4e7f36e1ef3f44f0b8470190d1e0969e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/d96d96c5f8b14c9eb2ff6446855e15cd, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/78318de9fbb44497b767eb1456113671, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/59b52f0f7a4845a2bd28521361a61156, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/684b71c669394fae82740f40bfad0a41, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/26d5b3e47127463a92069cb65327d6cf, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/f6e4b0474c7a44c9af434275f5064225] to archive 2024-12-08T08:17:04,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:17:04,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/45bffa0553ca45a2b03a75d9881a2adf to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/45bffa0553ca45a2b03a75d9881a2adf 2024-12-08T08:17:04,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/f8ced96bd6d0465abc1a6b91675390e6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/f8ced96bd6d0465abc1a6b91675390e6 2024-12-08T08:17:04,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/db3fc306ad284e79a66d2b8715f50d81 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/db3fc306ad284e79a66d2b8715f50d81 2024-12-08T08:17:04,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/1cd143815f6e456986dc0d8545021210 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/1cd143815f6e456986dc0d8545021210 2024-12-08T08:17:04,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/67d66acedccd459dad7e0f5d5c244d50 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/67d66acedccd459dad7e0f5d5c244d50 2024-12-08T08:17:04,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/6082cf6b5e0f45a18829db5d43da3d5e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/6082cf6b5e0f45a18829db5d43da3d5e 2024-12-08T08:17:04,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/279df1e57ffd4a1495820abd3a6bd99f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/279df1e57ffd4a1495820abd3a6bd99f 2024-12-08T08:17:04,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/df11d7aa70da4570a5097d50488050de to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/df11d7aa70da4570a5097d50488050de 2024-12-08T08:17:04,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/56b803a7d6744755bc7436b43d263fbc to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/56b803a7d6744755bc7436b43d263fbc 2024-12-08T08:17:04,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/c2716353b1c64be49be5ed9ec489918f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/c2716353b1c64be49be5ed9ec489918f 2024-12-08T08:17:04,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/2a1eddaa92224903b7e584a0b49c9fc3 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/2a1eddaa92224903b7e584a0b49c9fc3 2024-12-08T08:17:04,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/77484a26cdca457da7cb2e0933300800 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/77484a26cdca457da7cb2e0933300800 2024-12-08T08:17:04,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/4e7f36e1ef3f44f0b8470190d1e0969e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/4e7f36e1ef3f44f0b8470190d1e0969e 2024-12-08T08:17:05,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/d96d96c5f8b14c9eb2ff6446855e15cd to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/d96d96c5f8b14c9eb2ff6446855e15cd 2024-12-08T08:17:05,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/78318de9fbb44497b767eb1456113671 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/78318de9fbb44497b767eb1456113671 2024-12-08T08:17:05,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/59b52f0f7a4845a2bd28521361a61156 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/59b52f0f7a4845a2bd28521361a61156 2024-12-08T08:17:05,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/684b71c669394fae82740f40bfad0a41 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/684b71c669394fae82740f40bfad0a41 2024-12-08T08:17:05,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/26d5b3e47127463a92069cb65327d6cf to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/26d5b3e47127463a92069cb65327d6cf 2024-12-08T08:17:05,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/f6e4b0474c7a44c9af434275f5064225 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/f6e4b0474c7a44c9af434275f5064225 2024-12-08T08:17:05,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d249a2b374bf4a68ace156ae759de007, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/3db736c7b6cb4c3384b10899c0bfc5b0, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/8c193ce2e3f84acd8d7b1cdcbb7d56dc, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/9ca1b785cf5844a799daa32bb21ec423, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/b6b071e8b0d04ec598ddeb2f1800cde7, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/e8cffb43f7f04b79b80c1f3a9889b4b6, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ca8312c2de15437982aac36dc1a8f9ba, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d215dc0f908c4b69aa5962886249320b, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d403caf20c2a41e29a35830c1b90d9d2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ffb0266ae4aa4df0a412a7520c41faaa, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/086a3b1131c24aa9b2633cacabf25c61, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/364f79784d754f85923b8144e42d3a53, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/e97a428c05604cdcb1b00f4a8ff24767, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/221db69e1a5343969912176d26900953, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ec4a549f975e4186a0dea0f2ba510ff2, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/0f4ac45423194146b96eb1c88910fd5c, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/410b00ee6da94402a044b07074c04fac, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/b05e465e3a3740a6a9af457c792aad3e, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/56f0bd7561e24558a75563240cf0412a] to archive 2024-12-08T08:17:05,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T08:17:05,007 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d249a2b374bf4a68ace156ae759de007 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d249a2b374bf4a68ace156ae759de007 2024-12-08T08:17:05,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/3db736c7b6cb4c3384b10899c0bfc5b0 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/3db736c7b6cb4c3384b10899c0bfc5b0 2024-12-08T08:17:05,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/8c193ce2e3f84acd8d7b1cdcbb7d56dc to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/8c193ce2e3f84acd8d7b1cdcbb7d56dc 2024-12-08T08:17:05,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/9ca1b785cf5844a799daa32bb21ec423 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/9ca1b785cf5844a799daa32bb21ec423 2024-12-08T08:17:05,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/b6b071e8b0d04ec598ddeb2f1800cde7 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/b6b071e8b0d04ec598ddeb2f1800cde7 2024-12-08T08:17:05,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/e8cffb43f7f04b79b80c1f3a9889b4b6 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/e8cffb43f7f04b79b80c1f3a9889b4b6 2024-12-08T08:17:05,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ca8312c2de15437982aac36dc1a8f9ba to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ca8312c2de15437982aac36dc1a8f9ba 2024-12-08T08:17:05,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d215dc0f908c4b69aa5962886249320b to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d215dc0f908c4b69aa5962886249320b 2024-12-08T08:17:05,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d403caf20c2a41e29a35830c1b90d9d2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/d403caf20c2a41e29a35830c1b90d9d2 2024-12-08T08:17:05,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ffb0266ae4aa4df0a412a7520c41faaa to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ffb0266ae4aa4df0a412a7520c41faaa 2024-12-08T08:17:05,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/086a3b1131c24aa9b2633cacabf25c61 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/086a3b1131c24aa9b2633cacabf25c61 2024-12-08T08:17:05,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/364f79784d754f85923b8144e42d3a53 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/364f79784d754f85923b8144e42d3a53 2024-12-08T08:17:05,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/e97a428c05604cdcb1b00f4a8ff24767 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/e97a428c05604cdcb1b00f4a8ff24767 2024-12-08T08:17:05,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/221db69e1a5343969912176d26900953 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/221db69e1a5343969912176d26900953 2024-12-08T08:17:05,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ec4a549f975e4186a0dea0f2ba510ff2 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/ec4a549f975e4186a0dea0f2ba510ff2 2024-12-08T08:17:05,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/0f4ac45423194146b96eb1c88910fd5c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/0f4ac45423194146b96eb1c88910fd5c 2024-12-08T08:17:05,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/410b00ee6da94402a044b07074c04fac to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/410b00ee6da94402a044b07074c04fac 2024-12-08T08:17:05,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/b05e465e3a3740a6a9af457c792aad3e to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/b05e465e3a3740a6a9af457c792aad3e 2024-12-08T08:17:05,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/56f0bd7561e24558a75563240cf0412a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/56f0bd7561e24558a75563240cf0412a 2024-12-08T08:17:05,033 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/recovered.edits/354.seqid, newMaxSeqId=354, maxSeqId=4 2024-12-08T08:17:05,034 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. 2024-12-08T08:17:05,034 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for 634271208c7889374f3cf3905b95dab4: Waiting for close lock at 1733645822253Running coprocessor pre-close hooks at 1733645822253Disabling compacts and flushes for region at 1733645822253Disabling writes for close at 1733645823658 (+1405 ms)Obtaining lock to block concurrent updates at 1733645823658Preparing flush snapshotting stores in 634271208c7889374f3cf3905b95dab4 at 1733645823658Finished memstore snapshotting TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733645823659 (+1 ms)Flushing stores of TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4. at 1733645823659Flushing 634271208c7889374f3cf3905b95dab4/A: creating writer at 1733645823659Flushing mob file 634271208c7889374f3cf3905b95dab4/A: appending metadata at 1733645823662 (+3 ms)Flushing mob file 634271208c7889374f3cf3905b95dab4/A: closing flushed file at 1733645823662Flushing 634271208c7889374f3cf3905b95dab4/A: appending metadata at 1733645824075 (+413 ms)Flushing 634271208c7889374f3cf3905b95dab4/A: closing flushed file at 1733645824075Flushing 634271208c7889374f3cf3905b95dab4/B: creating writer at 1733645824085 (+10 ms)Flushing 634271208c7889374f3cf3905b95dab4/B: appending metadata at 1733645824088 (+3 ms)Flushing 634271208c7889374f3cf3905b95dab4/B: closing flushed file at 1733645824088Flushing 634271208c7889374f3cf3905b95dab4/C: creating writer at 1733645824504 (+416 ms)Flushing 634271208c7889374f3cf3905b95dab4/C: appending metadata at 1733645824509 (+5 ms)Flushing 634271208c7889374f3cf3905b95dab4/C: closing flushed file at 1733645824509Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51464161: reopening flushed file at 1733645824920 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b180a76: reopening flushed file at 1733645824927 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cdd6937: reopening flushed file at 1733645824932 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 634271208c7889374f3cf3905b95dab4 in 1311ms, sequenceid=351, compaction requested=true at 1733645824969 (+37 ms)Writing region close event to WAL at 1733645825029 (+60 ms)Running coprocessor post-close hooks at 1733645825034 (+5 ms)Closed at 1733645825034 2024-12-08T08:17:05,036 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed 634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:05,037 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=634271208c7889374f3cf3905b95dab4, regionState=CLOSED 2024-12-08T08:17:05,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 because future has completed 2024-12-08T08:17:05,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-08T08:17:05,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure 634271208c7889374f3cf3905b95dab4, server=422e41b75614,38619,1733645641981 in 2.9420 sec 2024-12-08T08:17:05,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-12-08T08:17:05,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=634271208c7889374f3cf3905b95dab4, UNASSIGN in 2.9460 sec 2024-12-08T08:17:05,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-08T08:17:05,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.9510 sec 2024-12-08T08:17:05,048 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733645825048"}]},"ts":"1733645825048"} 2024-12-08T08:17:05,050 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T08:17:05,050 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T08:17:05,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.9660 sec 2024-12-08T08:17:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-08T08:17:06,262 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:17:06,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-08T08:17:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:17:06,268 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:17:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-08T08:17:06,269 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:17:06,271 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,274 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C, FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/recovered.edits] 2024-12-08T08:17:06,277 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/528eeeca552d430aa502f0fbdc97fb69 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/528eeeca552d430aa502f0fbdc97fb69 2024-12-08T08:17:06,279 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/a607dd5a5bdf42f58c8755cd4479535f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/a607dd5a5bdf42f58c8755cd4479535f 2024-12-08T08:17:06,280 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/f50872a0eed44ecfb7dc76ca2a54849c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/f50872a0eed44ecfb7dc76ca2a54849c 2024-12-08T08:17:06,282 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/febbbeb624f146eaaa2dd3b9b969845f to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/A/febbbeb624f146eaaa2dd3b9b969845f 2024-12-08T08:17:06,285 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/92c414a9c5a340c4964542de553d39f8 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/92c414a9c5a340c4964542de553d39f8 2024-12-08T08:17:06,286 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/9e31b5f2cb5f4038b2260b7f38722606 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/9e31b5f2cb5f4038b2260b7f38722606 2024-12-08T08:17:06,288 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/aa35b3fadcd6419daea298534e11126c to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/aa35b3fadcd6419daea298534e11126c 2024-12-08T08:17:06,290 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/cc3925a5e1654ce082b52b3ef802bf2a to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/B/cc3925a5e1654ce082b52b3ef802bf2a 2024-12-08T08:17:06,292 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/5d8357c5b8784797a069d67d036a5471 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/5d8357c5b8784797a069d67d036a5471 2024-12-08T08:17:06,294 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/8561ffd9003a421fbe1c36fdd83a6f10 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/8561ffd9003a421fbe1c36fdd83a6f10 2024-12-08T08:17:06,295 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/91b21d29350a4b0ba259b419d432d4fd to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/91b21d29350a4b0ba259b419d432d4fd 2024-12-08T08:17:06,296 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/9420fa84578e4ace8812cd05e83b37e9 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/C/9420fa84578e4ace8812cd05e83b37e9 2024-12-08T08:17:06,299 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/recovered.edits/354.seqid to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4/recovered.edits/354.seqid 2024-12-08T08:17:06,300 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/default/TestAcidGuarantees/634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,300 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T08:17:06,300 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T08:17:06,301 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T08:17:06,302 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208189fde7058204da390f311861d099613_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208189fde7058204da390f311861d099613_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,303 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081fb4d8dbf86940aa8680ad125b5843c6_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081fb4d8dbf86940aa8680ad125b5843c6_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,304 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120821232d4f15b34470ba83b4e4b20d4608_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120821232d4f15b34470ba83b4e4b20d4608_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,304 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082270dd07845a477299c6435ebce2b047_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082270dd07845a477299c6435ebce2b047_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,305 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082d92896c1e1f40d08ca250e87ad951f2_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082d92896c1e1f40d08ca250e87ad951f2_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,306 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208401045d2241b43d4abbdf60f92cb3b2c_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208401045d2241b43d4abbdf60f92cb3b2c_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,306 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084f6625e10d9e42d6b2a8817ef53902fb_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084f6625e10d9e42d6b2a8817ef53902fb_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,307 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208580f21b093f34077a71c18f9db31fc28_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208580f21b093f34077a71c18f9db31fc28_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,308 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208583cdce18ee84689872714daee1d77f1_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208583cdce18ee84689872714daee1d77f1_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,308 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085ff3b9c203994d4b92f61ac01a297402_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085ff3b9c203994d4b92f61ac01a297402_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,309 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086786b65248b447f59158cdd9dbb96f89_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086786b65248b447f59158cdd9dbb96f89_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,310 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087f7cf72cd2ce4d37a8462112e6de656a_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412087f7cf72cd2ce4d37a8462112e6de656a_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,310 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088ec65641f64e41a09987feaf3c72dc6e_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088ec65641f64e41a09987feaf3c72dc6e_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,311 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a57ad792855c43ccb80c59c1bb736c0e_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a57ad792855c43ccb80c59c1bb736c0e_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,311 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208affb66ef32a84891bbc0444050679deb_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208affb66ef32a84891bbc0444050679deb_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,312 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208de5404c7569f4e71b87769b2b9993dbc_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208de5404c7569f4e71b87769b2b9993dbc_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,312 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e05d6893d0b64666b2ab310b5def3536_634271208c7889374f3cf3905b95dab4 to hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e05d6893d0b64666b2ab310b5def3536_634271208c7889374f3cf3905b95dab4 2024-12-08T08:17:06,313 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T08:17:06,314 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:17:06,316 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T08:17:06,318 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T08:17:06,319 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:17:06,319 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T08:17:06,319 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733645826319"}]},"ts":"9223372036854775807"} 2024-12-08T08:17:06,320 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T08:17:06,321 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 634271208c7889374f3cf3905b95dab4, NAME => 'TestAcidGuarantees,,1733645798725.634271208c7889374f3cf3905b95dab4.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T08:17:06,321 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T08:17:06,321 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733645826321"}]},"ts":"9223372036854775807"} 2024-12-08T08:17:06,322 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T08:17:06,323 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T08:17:06,324 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-12-08T08:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-08T08:17:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T08:17:06,382 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T08:17:06,394 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 247) - Thread LEAK? -, OpenFileDescriptor=455 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 398), ProcessCount=11 (was 11), AvailableMemoryMB=6589 (was 6964) 2024-12-08T08:17:06,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T08:17:06,395 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T08:17:06,395 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T08:17:06,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:06,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:06,395 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:17:06,395 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T08:17:06,395 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=350595032, stopped=false 2024-12-08T08:17:06,396 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=422e41b75614,44803,1733645641278 2024-12-08T08:17:06,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T08:17:06,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T08:17:06,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:17:06,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:17:06,466 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T08:17:06,467 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T08:17:06,468 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T08:17:06,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:06,468 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T08:17:06,468 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T08:17:06,469 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '422e41b75614,38619,1733645641981' ***** 2024-12-08T08:17:06,469 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T08:17:06,470 INFO [RS:0;422e41b75614:38619 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T08:17:06,470 INFO [RS:0;422e41b75614:38619 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T08:17:06,470 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T08:17:06,471 INFO [RS:0;422e41b75614:38619 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T08:17:06,471 INFO [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(959): stopping server 422e41b75614,38619,1733645641981 2024-12-08T08:17:06,471 INFO [RS:0;422e41b75614:38619 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T08:17:06,471 INFO [RS:0;422e41b75614:38619 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;422e41b75614:38619. 2024-12-08T08:17:06,471 DEBUG [RS:0;422e41b75614:38619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T08:17:06,471 DEBUG [RS:0;422e41b75614:38619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:06,471 INFO [RS:0;422e41b75614:38619 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T08:17:06,471 INFO [RS:0;422e41b75614:38619 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T08:17:06,471 INFO [RS:0;422e41b75614:38619 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T08:17:06,471 INFO [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T08:17:06,472 INFO [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T08:17:06,472 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-08T08:17:06,472 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T08:17:06,472 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T08:17:06,472 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T08:17:06,473 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T08:17:06,473 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T08:17:06,473 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T08:17:06,473 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-08T08:17:06,492 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/.tmp/info/fadb575744a14ad3a192ef44988b5e2b is 95, key is TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b./info:/1733645672068/DeleteFamily/seqid=0 2024-12-08T08:17:06,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742466_1642 (size=6514) 2024-12-08T08:17:06,556 INFO [regionserver/422e41b75614:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T08:17:06,673 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T08:17:06,854 INFO [regionserver/422e41b75614:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-08T08:17:06,855 INFO [regionserver/422e41b75614:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-08T08:17:06,873 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T08:17:06,897 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/.tmp/info/fadb575744a14ad3a192ef44988b5e2b 2024-12-08T08:17:06,922 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/.tmp/ns/7ef1e3c3e5274fd78e46c72d4e17d3e6 is 93, key is TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b./ns:/1733645672030/DeleteFamily/seqid=0 2024-12-08T08:17:06,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742467_1643 (size=5906) 2024-12-08T08:17:07,073 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T08:17:07,274 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T08:17:07,326 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/.tmp/ns/7ef1e3c3e5274fd78e46c72d4e17d3e6 2024-12-08T08:17:07,351 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/.tmp/rep_barrier/773add3f153344e5b3798ae85efd246c is 102, key is TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b./rep_barrier:/1733645672030/DeleteFamily/seqid=0 2024-12-08T08:17:07,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742468_1644 (size=6025) 2024-12-08T08:17:07,474 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T08:17:07,674 DEBUG [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T08:17:07,755 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/.tmp/rep_barrier/773add3f153344e5b3798ae85efd246c 2024-12-08T08:17:07,772 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/.tmp/table/909afa577bc64b45bd2c1549e667531c is 96, key is TestAcidGuarantees,,1733645646479.6e9f0e742ef7ddcc8f58c1d611d5e17b./table:/1733645672030/DeleteFamily/seqid=0 2024-12-08T08:17:07,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742469_1645 (size=5936) 2024-12-08T08:17:07,779 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/.tmp/table/909afa577bc64b45bd2c1549e667531c 2024-12-08T08:17:07,782 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/.tmp/info/fadb575744a14ad3a192ef44988b5e2b as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/info/fadb575744a14ad3a192ef44988b5e2b 2024-12-08T08:17:07,785 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/info/fadb575744a14ad3a192ef44988b5e2b, entries=12, sequenceid=90, filesize=6.4 K 2024-12-08T08:17:07,786 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/.tmp/ns/7ef1e3c3e5274fd78e46c72d4e17d3e6 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/ns/7ef1e3c3e5274fd78e46c72d4e17d3e6 2024-12-08T08:17:07,790 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/ns/7ef1e3c3e5274fd78e46c72d4e17d3e6, entries=8, sequenceid=90, filesize=5.8 K 2024-12-08T08:17:07,791 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/.tmp/rep_barrier/773add3f153344e5b3798ae85efd246c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/rep_barrier/773add3f153344e5b3798ae85efd246c 2024-12-08T08:17:07,794 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/rep_barrier/773add3f153344e5b3798ae85efd246c, entries=6, sequenceid=90, filesize=5.9 K 2024-12-08T08:17:07,795 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/.tmp/table/909afa577bc64b45bd2c1549e667531c as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/table/909afa577bc64b45bd2c1549e667531c 2024-12-08T08:17:07,799 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/table/909afa577bc64b45bd2c1549e667531c, entries=7, sequenceid=90, filesize=5.8 K 2024-12-08T08:17:07,800 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1327ms, sequenceid=90, compaction requested=false 2024-12-08T08:17:07,803 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-08T08:17:07,804 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T08:17:07,804 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T08:17:07,804 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733645826472Running coprocessor pre-close hooks at 1733645826472Disabling compacts and flushes for region at 1733645826472Disabling writes for close at 1733645826473 (+1 ms)Obtaining lock to block concurrent updates at 1733645826473Preparing flush snapshotting stores in 1588230740 at 1733645826473Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733645826473Flushing stores of hbase:meta,,1.1588230740 at 1733645826474 (+1 ms)Flushing 1588230740/info: creating writer at 1733645826474Flushing 1588230740/info: appending metadata at 1733645826492 (+18 ms)Flushing 1588230740/info: closing flushed file at 1733645826492Flushing 1588230740/ns: creating writer at 1733645826905 (+413 ms)Flushing 1588230740/ns: appending metadata at 1733645826922 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733645826922Flushing 1588230740/rep_barrier: creating writer at 1733645827333 (+411 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733645827351 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733645827351Flushing 1588230740/table: creating writer at 1733645827757 (+406 ms)Flushing 1588230740/table: appending metadata at 1733645827772 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733645827772Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d057a3a: reopening flushed file at 1733645827782 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32aa00c7: reopening flushed file at 1733645827785 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@490303fd: reopening flushed file at 1733645827790 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7af56f4c: reopening flushed file at 1733645827794 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1327ms, sequenceid=90, compaction requested=false at 1733645827800 (+6 ms)Writing region close event to WAL at 1733645827801 (+1 ms)Running coprocessor post-close hooks at 1733645827804 (+3 ms)Closed at 1733645827804 2024-12-08T08:17:07,804 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T08:17:07,874 INFO [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(976): stopping server 422e41b75614,38619,1733645641981; all regions closed. 2024-12-08T08:17:07,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741834_1010 (size=25344) 2024-12-08T08:17:07,880 DEBUG [RS:0;422e41b75614:38619 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/oldWALs 2024-12-08T08:17:07,881 INFO [RS:0;422e41b75614:38619 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 422e41b75614%2C38619%2C1733645641981.meta:.meta(num 1733645646088) 2024-12-08T08:17:07,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741832_1008 (size=15591649) 2024-12-08T08:17:07,883 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/WALs/422e41b75614,38619,1733645641981/422e41b75614%2C38619%2C1733645641981.1733645645013 not finished, retry = 0 2024-12-08T08:17:07,985 DEBUG [RS:0;422e41b75614:38619 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/oldWALs 2024-12-08T08:17:07,986 INFO [RS:0;422e41b75614:38619 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 422e41b75614%2C38619%2C1733645641981:(num 1733645645013) 2024-12-08T08:17:07,986 DEBUG [RS:0;422e41b75614:38619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:17:07,986 INFO [RS:0;422e41b75614:38619 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T08:17:07,986 INFO [RS:0;422e41b75614:38619 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T08:17:07,986 INFO [RS:0;422e41b75614:38619 {}] hbase.ChoreService(370): Chore service for: regionserver/422e41b75614:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-08T08:17:07,986 INFO [RS:0;422e41b75614:38619 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T08:17:07,986 INFO [regionserver/422e41b75614:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T08:17:07,987 INFO [RS:0;422e41b75614:38619 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38619 2024-12-08T08:17:08,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T08:17:08,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/422e41b75614,38619,1733645641981 2024-12-08T08:17:08,056 INFO [RS:0;422e41b75614:38619 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T08:17:08,144 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [422e41b75614,38619,1733645641981] 2024-12-08T08:17:08,238 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/422e41b75614,38619,1733645641981 already deleted, retry=false 2024-12-08T08:17:08,238 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 422e41b75614,38619,1733645641981 expired; onlineServers=0 2024-12-08T08:17:08,239 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '422e41b75614,44803,1733645641278' ***** 2024-12-08T08:17:08,239 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-08T08:17:08,239 INFO [M:0;422e41b75614:44803 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T08:17:08,239 INFO [M:0;422e41b75614:44803 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T08:17:08,239 DEBUG [M:0;422e41b75614:44803 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T08:17:08,240 DEBUG [M:0;422e41b75614:44803 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T08:17:08,240 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T08:17:08,240 DEBUG [master/422e41b75614:0:becomeActiveMaster-HFileCleaner.large.0-1733645644756 {}] cleaner.HFileCleaner(306): Exit Thread[master/422e41b75614:0:becomeActiveMaster-HFileCleaner.large.0-1733645644756,5,FailOnTimeoutGroup] 2024-12-08T08:17:08,240 DEBUG [master/422e41b75614:0:becomeActiveMaster-HFileCleaner.small.0-1733645644761 {}] cleaner.HFileCleaner(306): Exit Thread[master/422e41b75614:0:becomeActiveMaster-HFileCleaner.small.0-1733645644761,5,FailOnTimeoutGroup] 2024-12-08T08:17:08,241 INFO [M:0;422e41b75614:44803 {}] hbase.ChoreService(370): Chore service for: master/422e41b75614:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T08:17:08,241 INFO [M:0;422e41b75614:44803 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T08:17:08,241 DEBUG [M:0;422e41b75614:44803 {}] master.HMaster(1795): Stopping service threads 2024-12-08T08:17:08,242 INFO [M:0;422e41b75614:44803 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T08:17:08,242 INFO [M:0;422e41b75614:44803 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T08:17:08,243 ERROR [M:0;422e41b75614:44803 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-08T08:17:08,244 INFO [M:0;422e41b75614:44803 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T08:17:08,244 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T08:17:08,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T08:17:08,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38619-0x100047d29d00001, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T08:17:08,245 INFO [RS:0;422e41b75614:38619 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T08:17:08,245 INFO [RS:0;422e41b75614:38619 {}] regionserver.HRegionServer(1031): Exiting; stopping=422e41b75614,38619,1733645641981; zookeeper connection closed. 2024-12-08T08:17:08,246 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@37d9ee85 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@37d9ee85 2024-12-08T08:17:08,246 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-08T08:17:08,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T08:17:08,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:17:08,339 DEBUG [M:0;422e41b75614:44803 {}] zookeeper.ZKUtil(347): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-08T08:17:08,339 WARN [M:0;422e41b75614:44803 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-08T08:17:08,340 INFO [M:0;422e41b75614:44803 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/.lastflushedseqids 2024-12-08T08:17:08,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742470_1646 (size=72) 2024-12-08T08:17:08,749 INFO [M:0;422e41b75614:44803 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T08:17:08,749 INFO [M:0;422e41b75614:44803 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T08:17:08,749 DEBUG [M:0;422e41b75614:44803 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T08:17:08,750 INFO [M:0;422e41b75614:44803 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:17:08,750 DEBUG [M:0;422e41b75614:44803 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:17:08,750 DEBUG [M:0;422e41b75614:44803 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T08:17:08,750 DEBUG [M:0;422e41b75614:44803 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:17:08,750 INFO [M:0;422e41b75614:44803 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=750.26 KB heapSize=917.29 KB 2024-12-08T08:17:08,771 DEBUG [M:0;422e41b75614:44803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2dfb9411b38c4f399234e19c7519ba70 is 82, key is hbase:meta,,1/info:regioninfo/1733645646174/Put/seqid=0 2024-12-08T08:17:08,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742471_1647 (size=5672) 2024-12-08T08:17:09,175 INFO [M:0;422e41b75614:44803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2051 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2dfb9411b38c4f399234e19c7519ba70 2024-12-08T08:17:09,201 DEBUG [M:0;422e41b75614:44803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/836fa9014e32424e915cd8f44c61b943 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x93/proc:d/1733645801837/Put/seqid=0 2024-12-08T08:17:09,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742472_1648 (size=45693) 2024-12-08T08:17:09,606 INFO [M:0;422e41b75614:44803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=749.71 KB at sequenceid=2051 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/836fa9014e32424e915cd8f44c61b943 2024-12-08T08:17:09,616 INFO [M:0;422e41b75614:44803 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 836fa9014e32424e915cd8f44c61b943 2024-12-08T08:17:09,632 DEBUG [M:0;422e41b75614:44803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e6bf49f72b2640d790fbd96037629707 is 69, key is 422e41b75614,38619,1733645641981/rs:state/1733645644771/Put/seqid=0 2024-12-08T08:17:09,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073742473_1649 (size=5156) 2024-12-08T08:17:10,036 INFO [M:0;422e41b75614:44803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2051 (bloomFilter=true), to=hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e6bf49f72b2640d790fbd96037629707 2024-12-08T08:17:10,043 DEBUG [M:0;422e41b75614:44803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2dfb9411b38c4f399234e19c7519ba70 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2dfb9411b38c4f399234e19c7519ba70 2024-12-08T08:17:10,046 INFO [M:0;422e41b75614:44803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2dfb9411b38c4f399234e19c7519ba70, entries=8, sequenceid=2051, filesize=5.5 K 2024-12-08T08:17:10,047 DEBUG [M:0;422e41b75614:44803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/836fa9014e32424e915cd8f44c61b943 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/836fa9014e32424e915cd8f44c61b943 2024-12-08T08:17:10,050 INFO [M:0;422e41b75614:44803 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 836fa9014e32424e915cd8f44c61b943 2024-12-08T08:17:10,050 INFO [M:0;422e41b75614:44803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/836fa9014e32424e915cd8f44c61b943, entries=172, sequenceid=2051, filesize=44.6 K 2024-12-08T08:17:10,050 DEBUG [M:0;422e41b75614:44803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e6bf49f72b2640d790fbd96037629707 as hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e6bf49f72b2640d790fbd96037629707 2024-12-08T08:17:10,053 INFO [M:0;422e41b75614:44803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43591/user/jenkins/test-data/bb2e7083-a8e0-84e9-806d-2646ed93ad3b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e6bf49f72b2640d790fbd96037629707, entries=1, sequenceid=2051, filesize=5.0 K 2024-12-08T08:17:10,054 INFO [M:0;422e41b75614:44803 {}] regionserver.HRegion(3140): Finished flush of dataSize ~750.26 KB/768268, heapSize ~916.99 KB/939000, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1304ms, sequenceid=2051, compaction requested=false 2024-12-08T08:17:10,055 INFO [M:0;422e41b75614:44803 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:17:10,055 DEBUG [M:0;422e41b75614:44803 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733645828749Disabling compacts and flushes for region at 1733645828749Disabling writes for close at 1733645828750 (+1 ms)Obtaining lock to block concurrent updates at 1733645828750Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733645828750Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=768268, getHeapSize=939240, getOffHeapSize=0, getCellsCount=2248 at 1733645828750Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733645828751 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733645828751Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733645828770 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733645828770Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733645829183 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733645829201 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733645829201Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733645829616 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733645829632 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733645829632Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6867962f: reopening flushed file at 1733645830042 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@432feff7: reopening flushed file at 1733645830046 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@694a05c4: reopening flushed file at 1733645830050 (+4 ms)Finished flush of dataSize ~750.26 KB/768268, heapSize ~916.99 KB/939000, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1304ms, sequenceid=2051, compaction requested=false at 1733645830054 (+4 ms)Writing region close event to WAL at 1733645830055 (+1 ms)Closed at 1733645830055 2024-12-08T08:17:10,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41801 is added to blk_1073741830_1006 (size=902790) 2024-12-08T08:17:10,057 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T08:17:10,058 INFO [M:0;422e41b75614:44803 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T08:17:10,058 INFO [M:0;422e41b75614:44803 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44803 2024-12-08T08:17:10,058 INFO [M:0;422e41b75614:44803 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T08:17:10,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T08:17:10,206 INFO [M:0;422e41b75614:44803 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T08:17:10,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44803-0x100047d29d00000, quorum=127.0.0.1:59807, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T08:17:10,216 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T08:17:10,220 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T08:17:10,220 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T08:17:10,221 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T08:17:10,221 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/hadoop.log.dir/,STOPPED} 2024-12-08T08:17:10,224 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T08:17:10,224 WARN [BP-729112643-172.17.0.3-1733645636995 heartbeating to localhost/127.0.0.1:43591 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T08:17:10,224 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T08:17:10,224 WARN [BP-729112643-172.17.0.3-1733645636995 heartbeating to localhost/127.0.0.1:43591 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-729112643-172.17.0.3-1733645636995 (Datanode Uuid 760ec621-ddc7-40f2-983b-b5c7ee81a470) service to localhost/127.0.0.1:43591 2024-12-08T08:17:10,226 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/cluster_7f2d8a74-a01c-b592-2655-65641f2d4b95/data/data1/current/BP-729112643-172.17.0.3-1733645636995 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T08:17:10,226 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/cluster_7f2d8a74-a01c-b592-2655-65641f2d4b95/data/data2/current/BP-729112643-172.17.0.3-1733645636995 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T08:17:10,227 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T08:17:10,232 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T08:17:10,233 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T08:17:10,233 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T08:17:10,233 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T08:17:10,233 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/558361b9-3419-5419-c816-f21901664986/hadoop.log.dir/,STOPPED} 2024-12-08T08:17:10,249 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T08:17:10,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-08T08:17:10,554 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T08:17:10,556 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null